mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-18 22:33:42 +00:00
refactor: remove proxy API (#1596)
This commit is contained in:
98 files changed
+5958
-1225
No files matched your search
@@ -66,6 +66,7 @@ require (
|
|||||||
go.uber.org/automaxprocs v1.6.0 // indirect
|
go.uber.org/automaxprocs v1.6.0 // indirect
|
||||||
go.uber.org/mock v0.5.1 // indirect
|
go.uber.org/mock v0.5.1 // indirect
|
||||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||||
|
golang.org/x/mod v0.33.0
|
||||||
golang.org/x/oauth2 v0.35.0 // indirect
|
golang.org/x/oauth2 v0.35.0 // indirect
|
||||||
golang.org/x/sync v0.20.0 // indirect
|
golang.org/x/sync v0.20.0 // indirect
|
||||||
)
|
)
|
||||||
@@ -156,7 +157,6 @@ require (
|
|||||||
go.opentelemetry.io/otel/metric v1.39.0 // indirect
|
go.opentelemetry.io/otel/metric v1.39.0 // indirect
|
||||||
go.opentelemetry.io/otel/trace v1.39.0 // indirect
|
go.opentelemetry.io/otel/trace v1.39.0 // indirect
|
||||||
golang.org/x/crypto v0.48.0 // indirect
|
golang.org/x/crypto v0.48.0 // indirect
|
||||||
golang.org/x/mod v0.33.0 // indirect
|
|
||||||
golang.org/x/net v0.50.0
|
golang.org/x/net v0.50.0
|
||||||
golang.org/x/sys v0.44.0 // indirect
|
golang.org/x/sys v0.44.0 // indirect
|
||||||
golang.org/x/text v0.34.0
|
golang.org/x/text v0.34.0
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
package fetcher
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type TriggerSource string
|
||||||
|
|
||||||
|
const (
|
||||||
|
TriggerScheduled TriggerSource = "scheduled"
|
||||||
|
TriggerManual TriggerSource = "manual"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrJobSkipped = errors.New("protocol job skipped")
|
||||||
|
var ErrConfigChanged = errors.New("protocol configuration changed during execution")
|
||||||
|
|
||||||
|
// ConfigState embeds the common backend-owned state section in a protocol config.
|
||||||
|
// Public config PUTs always retain this section from the latest stored config.
|
||||||
|
type ConfigState[T any] struct {
|
||||||
|
State T `json:"state"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProtocolHost is scoped to one manager. UpdateState runs against the latest
|
||||||
|
// config under Downloader's lock; callbacks must not perform network I/O.
|
||||||
|
type ProtocolHost struct {
|
||||||
|
Read func(any) error
|
||||||
|
UpdateState func(func(config, state json.RawMessage) (any, error)) error
|
||||||
|
Fetch func(context.Context, string, int64, func([]byte) error) ([]byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ReadConfig[T any](host ProtocolHost) (*T, error) {
|
||||||
|
var cfg T
|
||||||
|
if err := host.Read(&cfg); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &cfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateState gives the callback a detached config snapshot and mutable state.
|
||||||
|
// Only the state is committed; edits to the config snapshot are discarded.
|
||||||
|
func UpdateState[C, S any](host ProtocolHost, update func(*C, *S) error) error {
|
||||||
|
return host.UpdateState(func(rawConfig, rawState json.RawMessage) (any, error) {
|
||||||
|
var cfg C
|
||||||
|
var state S
|
||||||
|
if err := json.Unmarshal(rawConfig, &cfg); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(rawState, &state); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := update(&cfg, &state); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &state, nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
type CronJob struct {
|
||||||
|
Name string
|
||||||
|
Every time.Duration
|
||||||
|
Timeout time.Duration
|
||||||
|
Run func(context.Context, TriggerSource) error
|
||||||
|
}
|
||||||
|
type CronProvider interface{ CronJobs(ProtocolHost) []CronJob }
|
||||||
|
|
||||||
|
// ConfigNormalizer is optional. It owns protocol validation and derived fields.
|
||||||
|
type ConfigNormalizer interface {
|
||||||
|
NormalizeConfig(json.RawMessage) (any, error)
|
||||||
|
}
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
# GitHub mirror execution
|
||||||
|
|
||||||
|
This package owns GitHub URL rewriting and execution strategies. It depends only
|
||||||
|
on the standard library and does not depend on Downloader, HTTP clients, Git, or
|
||||||
|
storage. Downloader persists `Config` at its top-level `githubMirror` field;
|
||||||
|
`pkg/base` preserves the existing configuration names through aliases.
|
||||||
|
|
||||||
|
## Race: resource reads
|
||||||
|
|
||||||
|
```go
|
||||||
|
body, err := githubmirror.Race(ctx, cfg.GithubMirror, rawURL,
|
||||||
|
githubmirror.Options[[]byte]{
|
||||||
|
Kind: githubmirror.Resource,
|
||||||
|
Timeout: 30 * time.Second,
|
||||||
|
},
|
||||||
|
func(ctx context.Context, candidate string) ([]byte, error) {
|
||||||
|
return fetchAndValidate(ctx, client, candidate)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
The callback owns HTTP requests, size limits, and content validation. The first
|
||||||
|
completed valid result wins. Other operations are canceled and drained. There is
|
||||||
|
no HEAD-probe assumption or client-specific HTTP behavior inside the executor.
|
||||||
|
|
||||||
|
## Fallback: repository operations
|
||||||
|
|
||||||
|
```go
|
||||||
|
repo, err := githubmirror.Fallback(ctx, cfg.GithubMirror, officialURL,
|
||||||
|
githubmirror.Options[*git.Repository]{
|
||||||
|
Kind: githubmirror.Repository,
|
||||||
|
},
|
||||||
|
func(ctx context.Context, candidate string) (*git.Repository, error) {
|
||||||
|
// Use a clean temporary checkout for each attempt.
|
||||||
|
return clone(ctx, candidate)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
The official URL is attempted first; mirrors follow their configured order.
|
||||||
|
jsDelivr is excluded for repositories. Stop on the first success. Perform
|
||||||
|
persistent installation or other business state changes after success, rather
|
||||||
|
than inside a potentially repeated attempt.
|
||||||
|
|
||||||
|
## Results with resources
|
||||||
|
|
||||||
|
For temporary files, set `Options[*Download].Discard` to remove the file. Return
|
||||||
|
the partial `*Download` alongside errors, so cleanup also covers failed reads or
|
||||||
|
validation. The executor discards every failure and every race loser exactly
|
||||||
|
once. The caller owns and must track/release the winning result.
|
||||||
|
|
||||||
|
- `Timeout == 0` uses 30 seconds per resource attempt and no overall deadline
|
||||||
|
for repository attempts; a negative timeout is invalid. Extension Git HTTP
|
||||||
|
transport limits connections/TLS handshakes to 10 seconds and resets a 60-second
|
||||||
|
read deadline as data arrives. Slow continuous transfers can finish, while
|
||||||
|
stalled sources fail and allow ordered fallback. Parent cancellation still applies.
|
||||||
|
- Attempts must honor the supplied context; draining waits for them to return.
|
||||||
|
- Attempt contexts are canceled after return, so callbacks must finish reading
|
||||||
|
HTTP bodies before returning. Returning a still-streaming response is unsupported.
|
||||||
|
- Discard callbacks must accept zero values, must not panic, and run serially.
|
||||||
|
- Parent cancellation wins over a result selected while the parent is canceled;
|
||||||
|
the selected result is then discarded as well.
|
||||||
|
- Errors preserve their causes through wrapping/joining for `errors.Is`.
|
||||||
|
- `github.com` and `raw.githubusercontent.com` expand into eligible mirrors.
|
||||||
|
jsDelivr is only eligible for raw source resources. `api.github.com` is never
|
||||||
|
mirrored and is rejected by the public mirror candidates API.
|
||||||
|
- Client application update checks request the official GitHub API directly,
|
||||||
|
falling back to the independent Gopeed website API on request or validation
|
||||||
|
failure. Only package downloads obtain URLs from
|
||||||
|
`POST /api/v1/mirrors/github/candidates`. Candidate generation never performs
|
||||||
|
HTTP requests.
|
||||||
|
- Disabled mirrors and other URLs execute only the original URL.
|
||||||
|
- Config is a caller-owned immutable snapshot for the duration of a call.
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
// Package githubmirror rewrites GitHub URLs and executes mirrored operations.
|
||||||
|
package githubmirror
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Mirror struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
IsBuiltIn bool `json:"isBuiltIn"`
|
||||||
|
IsDeleted bool `json:"isDeleted"`
|
||||||
|
}
|
||||||
|
type Config struct {
|
||||||
|
Enabled bool `json:"enabled"`
|
||||||
|
Mirrors []Mirror `json:"mirrors"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func DefaultConfig() *Config {
|
||||||
|
return &Config{Enabled: true, Mirrors: []Mirror{
|
||||||
|
{Type: "jsdelivr", URL: "https://fastly.jsdelivr.net/gh", IsBuiltIn: true},
|
||||||
|
{Type: "ghProxy", URL: "https://fastgit.cc", IsBuiltIn: true},
|
||||||
|
{Type: "ghProxy", URL: "https://gh-proxy.com", IsBuiltIn: true},
|
||||||
|
{Type: "ghProxy", URL: "https://ghfast.top", IsBuiltIn: true},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
func (c *Config) Validate() error {
|
||||||
|
if len(c.Mirrors) > 32 {
|
||||||
|
return fmt.Errorf("too many GitHub mirrors")
|
||||||
|
}
|
||||||
|
for _, mirror := range c.Mirrors {
|
||||||
|
u, err := url.Parse(mirror.URL)
|
||||||
|
if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
|
||||||
|
return fmt.Errorf("invalid GitHub mirror URL")
|
||||||
|
}
|
||||||
|
if mirror.Type != "jsdelivr" && mirror.Type != "ghProxy" {
|
||||||
|
return fmt.Errorf("invalid GitHub mirror type: %s", mirror.Type)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kind describes what a candidate endpoint must support.
|
||||||
|
type Kind uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
Resource Kind = iota
|
||||||
|
Repository
|
||||||
|
)
|
||||||
|
|
||||||
|
// Candidates preserves the official URL first. jsDelivr is only eligible for
|
||||||
|
// raw source resources; Git operations use mirrors supporting Git Smart HTTP.
|
||||||
|
// GitHub API URLs are never mirrored.
|
||||||
|
func Candidates(c *Config, raw string, kind Kind) []string {
|
||||||
|
result := []string{raw}
|
||||||
|
u, err := url.Parse(raw)
|
||||||
|
if err != nil || c == nil || !c.Enabled || u.User != nil || (u.Scheme != "https" && u.Scheme != "http") {
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
if u.Host != "github.com" && u.Host != "raw.githubusercontent.com" {
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
seen := map[string]bool{raw: true}
|
||||||
|
for _, mirror := range c.Mirrors {
|
||||||
|
if mirror.IsDeleted {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var candidate string
|
||||||
|
switch mirror.Type {
|
||||||
|
case "ghProxy":
|
||||||
|
candidate = strings.TrimRight(mirror.URL, "/") + "/" + raw
|
||||||
|
case "jsdelivr":
|
||||||
|
if kind == Repository || u.Host != "raw.githubusercontent.com" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
parts := strings.SplitN(strings.TrimPrefix(u.EscapedPath(), "/"), "/", 4)
|
||||||
|
if len(parts) != 4 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
candidate = strings.TrimRight(mirror.URL, "/") + "/" + parts[0] + "/" + parts[1] + "@" + parts[2] + "/" + parts[3]
|
||||||
|
}
|
||||||
|
if candidate != "" && !seen[candidate] {
|
||||||
|
result = append(result, candidate)
|
||||||
|
seen[candidate] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
package githubmirror
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMirrorCandidates(t *testing.T) {
|
||||||
|
cfg := &Config{Enabled: true, Mirrors: []Mirror{
|
||||||
|
{Type: "jsdelivr", URL: "https://cdn.example/gh/"},
|
||||||
|
{Type: "ghProxy", URL: "https://mirror.example/"},
|
||||||
|
{Type: "ghProxy", URL: "https://deleted.example", IsDeleted: true},
|
||||||
|
}}
|
||||||
|
source := "https://raw.githubusercontent.com/owner/repo/main/a.txt"
|
||||||
|
want := []string{source, "https://cdn.example/gh/owner/repo@main/a.txt", "https://mirror.example/" + source}
|
||||||
|
if got := Candidates(cfg, source, Resource); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("got %v", got)
|
||||||
|
}
|
||||||
|
repo := "https://github.com/owner/repo.git"
|
||||||
|
if got := Candidates(cfg, repo, Repository); !reflect.DeepEqual(got, []string{repo, "https://mirror.example/" + repo}) {
|
||||||
|
t.Fatalf("git candidates %v", got)
|
||||||
|
}
|
||||||
|
for _, raw := range []string{"https://github.com.evil.example/a", "https://example.com/a", "https://user:[email protected]/a"} {
|
||||||
|
if got := Candidates(cfg, raw, Resource); !reflect.DeepEqual(got, []string{raw}) {
|
||||||
|
t.Fatalf("unexpected mirror: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
apiURL := "https://api.github.com/repos/GopeedLab/gopeed/releases?per_page=10"
|
||||||
|
for _, kind := range []Kind{Resource, Repository} {
|
||||||
|
if got := Candidates(cfg, apiURL, kind); !reflect.DeepEqual(got, []string{apiURL}) {
|
||||||
|
t.Fatalf("GitHub API must never use mirrors: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cfg.Enabled = false
|
||||||
|
if got := Candidates(cfg, source, Resource); !reflect.DeepEqual(got, []string{source}) {
|
||||||
|
t.Fatalf("disabled mirror: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
package githubmirror
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Options configures execution, not URL persistence. Discard releases results
|
||||||
|
// from failed attempts and successful race losers, including partial results
|
||||||
|
// returned together with an error. The caller owns the winning result.
|
||||||
|
type Options[T any] struct {
|
||||||
|
Kind Kind
|
||||||
|
// Timeout limits each attempt. Zero uses 30 seconds for resources and no
|
||||||
|
// overall deadline for repositories, whose transport owns idle timeouts.
|
||||||
|
Timeout time.Duration
|
||||||
|
Discard func(T)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attempt must honor ctx and return a completed, validated result. Its context
|
||||||
|
// is canceled after it returns, so live response streams must not be returned.
|
||||||
|
// Return any partially acquired resources along with errors so Discard can
|
||||||
|
// release them. Failed attempts must not mutate persistent business state.
|
||||||
|
type Attempt[T any] func(context.Context, string) (T, error)
|
||||||
|
|
||||||
|
const defaultTimeout = 30 * time.Second
|
||||||
|
|
||||||
|
func prepare[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) ([]string, time.Duration, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
if raw == "" {
|
||||||
|
return nil, 0, errors.New("empty GitHub source URL")
|
||||||
|
}
|
||||||
|
if options.Kind != Resource && options.Kind != Repository {
|
||||||
|
return nil, 0, errors.New("invalid GitHub resource kind")
|
||||||
|
}
|
||||||
|
if options.Timeout < 0 {
|
||||||
|
return nil, 0, errors.New("invalid GitHub attempt timeout")
|
||||||
|
}
|
||||||
|
if attempt == nil {
|
||||||
|
return nil, 0, errors.New("GitHub mirror attempt is required")
|
||||||
|
}
|
||||||
|
if config != nil {
|
||||||
|
if err := config.Validate(); err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
timeout := options.Timeout
|
||||||
|
if timeout == 0 && options.Kind == Resource {
|
||||||
|
timeout = defaultTimeout
|
||||||
|
}
|
||||||
|
return Candidates(config, raw, options.Kind), timeout, nil
|
||||||
|
}
|
||||||
|
func invoke[T any](ctx context.Context, timeout time.Duration, candidate string, attempt Attempt[T]) (value T, err error) {
|
||||||
|
var cancel context.CancelFunc
|
||||||
|
if timeout > 0 {
|
||||||
|
ctx, cancel = context.WithTimeout(ctx, timeout)
|
||||||
|
} else {
|
||||||
|
ctx, cancel = context.WithCancel(ctx)
|
||||||
|
}
|
||||||
|
defer cancel()
|
||||||
|
defer func() {
|
||||||
|
if recovered := recover(); recovered != nil {
|
||||||
|
err = fmt.Errorf("GitHub mirror attempt panicked: %v", recovered)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
value, err = attempt(ctx, candidate)
|
||||||
|
if err == nil {
|
||||||
|
err = ctx.Err()
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
func discard[T any](options Options[T], value T) {
|
||||||
|
if options.Discard != nil {
|
||||||
|
options.Discard(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Race executes the official URL and eligible mirrors concurrently. The first
|
||||||
|
// completed successful result wins; losing attempts are canceled and drained
|
||||||
|
// before return so no temporary files or other resources outlive the call.
|
||||||
|
// HTTP status/content validation is the attempt's responsibility.
|
||||||
|
func Race[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) (T, error) {
|
||||||
|
var zero T
|
||||||
|
candidates, timeout, err := prepare(ctx, config, raw, options, attempt)
|
||||||
|
if err != nil {
|
||||||
|
return zero, err
|
||||||
|
}
|
||||||
|
raceCtx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
type outcome struct {
|
||||||
|
index int
|
||||||
|
value T
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
outcomes := make(chan outcome, len(candidates))
|
||||||
|
for index, candidate := range candidates {
|
||||||
|
go func(index int, candidate string) {
|
||||||
|
value, err := invoke(raceCtx, timeout, candidate, attempt)
|
||||||
|
outcomes <- outcome{index, value, err}
|
||||||
|
}(index, candidate)
|
||||||
|
}
|
||||||
|
var winner T
|
||||||
|
won := false
|
||||||
|
failures := make([]error, len(candidates))
|
||||||
|
for range candidates {
|
||||||
|
outcome := <-outcomes
|
||||||
|
if !won && outcome.err == nil && ctx.Err() == nil {
|
||||||
|
winner = outcome.value
|
||||||
|
won = true
|
||||||
|
cancel()
|
||||||
|
} else {
|
||||||
|
discard(options, outcome.value)
|
||||||
|
if outcome.err != nil {
|
||||||
|
failures[outcome.index] = fmt.Errorf("GitHub source %d: %w", outcome.index+1, outcome.err)
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
if won {
|
||||||
|
discard(options, winner)
|
||||||
|
}
|
||||||
|
return zero, err
|
||||||
|
}
|
||||||
|
if won {
|
||||||
|
return winner, nil
|
||||||
|
}
|
||||||
|
return zero, fmt.Errorf("all GitHub sources failed: %w", errors.Join(failures...))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback executes the official URL first, then eligible mirrors in their
|
||||||
|
// configured order. It stops immediately on success and cleans every failure.
|
||||||
|
func Fallback[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) (T, error) {
|
||||||
|
var zero T
|
||||||
|
candidates, timeout, err := prepare(ctx, config, raw, options, attempt)
|
||||||
|
if err != nil {
|
||||||
|
return zero, err
|
||||||
|
}
|
||||||
|
failures := make([]error, 0, len(candidates))
|
||||||
|
for index, candidate := range candidates {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return zero, err
|
||||||
|
}
|
||||||
|
value, err := invoke(ctx, timeout, candidate, attempt)
|
||||||
|
if err == nil && ctx.Err() == nil {
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
discard(options, value)
|
||||||
|
if parentErr := ctx.Err(); parentErr != nil {
|
||||||
|
return zero, parentErr
|
||||||
|
}
|
||||||
|
failures = append(failures, fmt.Errorf("GitHub source %d: %w", index+1, err))
|
||||||
|
}
|
||||||
|
return zero, fmt.Errorf("all GitHub sources failed: %w", errors.Join(failures...))
|
||||||
|
}
|
||||||
@@ -0,0 +1,214 @@
|
|||||||
|
package githubmirror
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const testSource = "https://github.com/owner/repo.git"
|
||||||
|
|
||||||
|
func testConfig() *Config {
|
||||||
|
return &Config{Enabled: true, Mirrors: []Mirror{
|
||||||
|
{Type: "ghProxy", URL: "https://first.example"},
|
||||||
|
{Type: "ghProxy", URL: "https://second.example"},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
func TestRaceWinsAndDrainsOwnedResults(t *testing.T) {
|
||||||
|
config := testConfig()
|
||||||
|
candidates := Candidates(config, testSource, Resource)
|
||||||
|
var started sync.WaitGroup
|
||||||
|
started.Add(len(candidates))
|
||||||
|
discarded := []string{}
|
||||||
|
winner, err := Race(context.Background(), config, testSource, Options[string]{
|
||||||
|
Discard: func(value string) { discarded = append(discarded, value) },
|
||||||
|
}, func(ctx context.Context, address string) (string, error) {
|
||||||
|
started.Done()
|
||||||
|
started.Wait()
|
||||||
|
switch address {
|
||||||
|
case candidates[0]:
|
||||||
|
return address, errors.New("official offline")
|
||||||
|
case candidates[1]:
|
||||||
|
return address, nil
|
||||||
|
default:
|
||||||
|
<-ctx.Done()
|
||||||
|
return address, ctx.Err()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if err != nil || winner != candidates[1] {
|
||||||
|
t.Fatalf("winner=%s err=%v", winner, err)
|
||||||
|
}
|
||||||
|
if len(discarded) != 2 || !contains(discarded, candidates[0]) || !contains(discarded, candidates[2]) {
|
||||||
|
t.Fatalf("discarded %v", discarded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func contains(values []string, value string) bool {
|
||||||
|
for _, item := range values {
|
||||||
|
if item == value {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
func TestRaceCleansSimultaneousSuccessfulLosers(t *testing.T) {
|
||||||
|
config := testConfig()
|
||||||
|
var started sync.WaitGroup
|
||||||
|
started.Add(3)
|
||||||
|
cleaned := []string{}
|
||||||
|
winner, err := Race(context.Background(), config, testSource, Options[string]{Discard: func(value string) { cleaned = append(cleaned, value) }}, func(_ context.Context, address string) (string, error) {
|
||||||
|
started.Done()
|
||||||
|
started.Wait()
|
||||||
|
return address, nil
|
||||||
|
})
|
||||||
|
if err != nil || winner == "" || len(cleaned) != 2 || contains(cleaned, winner) {
|
||||||
|
t.Fatalf("winner=%s discarded=%v err=%v", winner, cleaned, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestRaceParentCancellationCleansWinnerToo(t *testing.T) {
|
||||||
|
config := testConfig()
|
||||||
|
config.Mirrors = config.Mirrors[:1]
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
cleaned := []string{}
|
||||||
|
winner, err := Race(ctx, config, testSource, Options[string]{Discard: func(value string) { cleaned = append(cleaned, value) }}, func(ctx context.Context, address string) (string, error) {
|
||||||
|
if address == testSource {
|
||||||
|
return address, nil
|
||||||
|
}
|
||||||
|
<-ctx.Done()
|
||||||
|
cancel()
|
||||||
|
return address, ctx.Err()
|
||||||
|
})
|
||||||
|
if !errors.Is(err, context.Canceled) || winner != "" || len(cleaned) != 2 || !contains(cleaned, testSource) {
|
||||||
|
t.Fatalf("winner=%s cleanup=%v err=%v", winner, cleaned, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestRaceTimeoutAndErrorAggregation(t *testing.T) {
|
||||||
|
expected := errors.New("offline")
|
||||||
|
config := testConfig()
|
||||||
|
config.Mirrors = config.Mirrors[:1]
|
||||||
|
discarded := 0
|
||||||
|
_, err := Race(context.Background(), config, testSource, Options[string]{Timeout: 10 * time.Millisecond, Discard: func(string) { discarded++ }}, func(ctx context.Context, address string) (string, error) {
|
||||||
|
if address == testSource {
|
||||||
|
return address, expected
|
||||||
|
}
|
||||||
|
<-ctx.Done()
|
||||||
|
return address, ctx.Err()
|
||||||
|
})
|
||||||
|
if !errors.Is(err, expected) || !errors.Is(err, context.DeadlineExceeded) || discarded != 2 {
|
||||||
|
t.Fatalf("discarded=%d err=%v", discarded, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestFallbackPreservesOrderAndStopsOnSuccess(t *testing.T) {
|
||||||
|
config := testConfig()
|
||||||
|
candidates := Candidates(config, testSource, Repository)
|
||||||
|
tried := []string{}
|
||||||
|
cleaned := []string{}
|
||||||
|
winner, err := Fallback(context.Background(), config, testSource, Options[string]{Kind: Repository, Discard: func(value string) { cleaned = append(cleaned, value) }}, func(_ context.Context, address string) (string, error) {
|
||||||
|
tried = append(tried, address)
|
||||||
|
if address == candidates[1] {
|
||||||
|
return address, nil
|
||||||
|
}
|
||||||
|
return address, errors.New("offline")
|
||||||
|
})
|
||||||
|
if err != nil || winner != candidates[1] || !reflect.DeepEqual(tried, candidates[:2]) || !reflect.DeepEqual(cleaned, candidates[:1]) {
|
||||||
|
t.Fatalf("winner=%s tried=%v discarded=%v err=%v", winner, tried, cleaned, err)
|
||||||
|
}
|
||||||
|
tried = nil
|
||||||
|
_, err = Fallback(context.Background(), config, testSource, Options[string]{Kind: Repository}, func(_ context.Context, address string) (string, error) {
|
||||||
|
tried = append(tried, address)
|
||||||
|
return address, nil
|
||||||
|
})
|
||||||
|
if err != nil || !reflect.DeepEqual(tried, candidates[:1]) {
|
||||||
|
t.Fatalf("tried=%v err=%v", tried, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestFallbackTimeoutContinuesWithFreshContext(t *testing.T) {
|
||||||
|
discarded := []string{}
|
||||||
|
winner, err := Fallback(context.Background(), testConfig(), testSource, Options[string]{Kind: Repository, Timeout: 10 * time.Millisecond, Discard: func(value string) { discarded = append(discarded, value) }}, func(ctx context.Context, address string) (string, error) {
|
||||||
|
if address == testSource {
|
||||||
|
<-ctx.Done()
|
||||||
|
return address, ctx.Err()
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return address, err
|
||||||
|
}
|
||||||
|
return address, nil
|
||||||
|
})
|
||||||
|
if err != nil || !strings.HasPrefix(winner, "https://first.example/") || !reflect.DeepEqual(discarded, []string{testSource}) {
|
||||||
|
t.Fatalf("winner=%s cleanup=%v err=%v", winner, discarded, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestFallbackCancellationStopsBeforeNextMirror(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
calls := 0
|
||||||
|
cleaned := 0
|
||||||
|
_, err := Fallback(ctx, testConfig(), testSource, Options[string]{Kind: Repository, Discard: func(string) { cleaned++ }}, func(ctx context.Context, address string) (string, error) {
|
||||||
|
calls++
|
||||||
|
cancel()
|
||||||
|
return address, ctx.Err()
|
||||||
|
})
|
||||||
|
if !errors.Is(err, context.Canceled) || calls != 1 || cleaned != 1 {
|
||||||
|
t.Fatalf("calls=%d discarded=%d err=%v", calls, cleaned, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestExecutorsDoNotStartInvalidOrCanceledRequests(t *testing.T) {
|
||||||
|
canceled, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
for _, test := range []struct {
|
||||||
|
ctx context.Context
|
||||||
|
config *Config
|
||||||
|
options Options[string]
|
||||||
|
attempt Attempt[string]
|
||||||
|
}{
|
||||||
|
{ctx: canceled, options: Options[string]{}, attempt: func(context.Context, string) (string, error) { t.Fatal("called after cancellation"); return "", nil }},
|
||||||
|
{ctx: context.Background(), options: Options[string]{Kind: 99}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid kind"); return "", nil }},
|
||||||
|
{ctx: context.Background(), options: Options[string]{Timeout: -time.Second}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid timeout"); return "", nil }},
|
||||||
|
{ctx: context.Background(), options: Options[string]{}, attempt: nil},
|
||||||
|
{ctx: context.Background(), config: &Config{Mirrors: []Mirror{{Type: "invalid", URL: "https://mirror.example"}}}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid config"); return "", nil }},
|
||||||
|
} {
|
||||||
|
if _, err := Race(test.ctx, test.config, testSource, test.options, test.attempt); err == nil {
|
||||||
|
t.Fatal("invalid race accepted")
|
||||||
|
}
|
||||||
|
if _, err := Fallback(test.ctx, test.config, testSource, test.options, test.attempt); err == nil {
|
||||||
|
t.Fatal("invalid fallback accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestAttemptPanicBecomesError(t *testing.T) {
|
||||||
|
attempt := func(context.Context, string) (string, error) { panic("broken operation") }
|
||||||
|
if _, err := Race(context.Background(), nil, testSource, Options[string]{}, attempt); err == nil || !strings.Contains(err.Error(), "broken operation") {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := Fallback(context.Background(), nil, testSource, Options[string]{}, attempt); err == nil || !strings.Contains(err.Error(), "broken operation") {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestNonGithubResourcesAndDisabledMirrorsUseOnlyOriginal(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
config *Config
|
||||||
|
raw string
|
||||||
|
}{{testConfig(), "https://example.com/resource"}, {&Config{Enabled: false, Mirrors: testConfig().Mirrors}, testSource}} {
|
||||||
|
calls := 0
|
||||||
|
got, err := Race(context.Background(), test.config, test.raw, Options[string]{}, func(_ context.Context, address string) (string, error) { calls++; return address, nil })
|
||||||
|
if err != nil || got != test.raw || calls != 1 {
|
||||||
|
t.Fatalf("got=%s calls=%d err=%v", got, calls, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRepositoryAttemptsHaveNoDefaultDeadline(t *testing.T) {
|
||||||
|
value, err := Fallback(context.Background(), nil, testSource, Options[string]{Kind: Repository}, func(ctx context.Context, raw string) (string, error) {
|
||||||
|
if _, ok := ctx.Deadline(); ok {
|
||||||
|
t.Fatal("repository attempt has a total deadline")
|
||||||
|
}
|
||||||
|
return raw, nil
|
||||||
|
})
|
||||||
|
if err != nil || value != testSource {
|
||||||
|
t.Fatalf("value=%s error=%v", value, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,8 +1,29 @@
|
|||||||
package bt
|
package bt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
|
)
|
||||||
|
|
||||||
|
type trackerSubscriptionStatus struct {
|
||||||
|
Success bool `json:"success"`
|
||||||
|
TrackerCount int `json:"trackerCount"`
|
||||||
|
LastFetchTime time.Time `json:"lastFetchTime"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type state struct {
|
||||||
|
TrackerSubscriptionStatuses map[string]trackerSubscriptionStatus `json:"trackerSubscriptionStatuses"`
|
||||||
|
TrackerSources map[string][]string `json:"trackerSources"`
|
||||||
|
LastTrackerUpdateTime time.Time `json:"lastTrackerUpdateTime"`
|
||||||
|
}
|
||||||
|
|
||||||
type config struct {
|
type config struct {
|
||||||
ListenPort int `json:"listenPort"`
|
fetcher.ConfigState[state]
|
||||||
Trackers []string `json:"trackers"`
|
TrackerSubscribeURLs []string `json:"trackerSubscribeUrls"`
|
||||||
|
AutoUpdateTrackers bool `json:"autoUpdateTrackers"`
|
||||||
|
CustomTrackers []string `json:"customTrackers"`
|
||||||
|
ListenPort int `json:"listenPort"`
|
||||||
// SeedKeep is always keep seeding after downloading is complete, unless manually stopped.
|
// SeedKeep is always keep seeding after downloading is complete, unless manually stopped.
|
||||||
SeedKeep bool `json:"seedKeep"`
|
SeedKeep bool `json:"seedKeep"`
|
||||||
// SeedRatio is the ratio of uploaded data to downloaded data to seed.
|
// SeedRatio is the ratio of uploaded data to downloaded data to seed.
|
||||||
|
|||||||
@@ -198,7 +198,7 @@ func TestBTDownloadSources(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestBTDirectDownloadWithExtraTrackers(t *testing.T) {
|
func TestBTDirectDownloadWithExtraTrackers(t *testing.T) {
|
||||||
for _, source := range []string{"request", "config"} {
|
for _, source := range []string{"request", "config", "subscription"} {
|
||||||
t.Run(source, func(t *testing.T) {
|
t.Run(source, func(t *testing.T) {
|
||||||
fixture := makeBTFixture(t, map[string][]byte{
|
fixture := makeBTFixture(t, map[string][]byte{
|
||||||
"a.bin": btPayload(20*1024, "direct-a"),
|
"a.bin": btPayload(20*1024, "direct-a"),
|
||||||
@@ -217,7 +217,23 @@ func TestBTDirectDownloadWithExtraTrackers(t *testing.T) {
|
|||||||
if source == "request" {
|
if source == "request" {
|
||||||
req.Extra = &btprotocol.ReqExtra{Trackers: []string{fixture.TrackerURL}}
|
req.Extra = &btprotocol.ReqExtra{Trackers: []string{fixture.TrackerURL}}
|
||||||
} else {
|
} else {
|
||||||
f.config.Trackers = []string{fixture.TrackerURL}
|
backendConfig := new(FetcherManager).DefaultConfig().(*config)
|
||||||
|
if source == "config" {
|
||||||
|
backendConfig.CustomTrackers = []string{fixture.TrackerURL}
|
||||||
|
} else {
|
||||||
|
backendConfig.TrackerSubscribeURLs = []string{"https://subscription.example/list"}
|
||||||
|
backendConfig.State.TrackerSources[backendConfig.TrackerSubscribeURLs[0]] = []string{fixture.TrackerURL}
|
||||||
|
}
|
||||||
|
f.ctl.GetConfig = func(dst any) {
|
||||||
|
raw, err := json.Marshal(backendConfig)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, dst); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// CreateDirect supplies options before Downloader resolves the task.
|
// CreateDirect supplies options before Downloader resolves the task.
|
||||||
dir := filepath.Join(t.TempDir(), "direct-download")
|
dir := filepath.Join(t.TempDir(), "direct-download")
|
||||||
|
|||||||
@@ -744,6 +744,7 @@ func (f *Fetcher) WaitUpload() (err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) addTorrent(req *base.Request) (err error) {
|
func (f *Fetcher) addTorrent(req *base.Request) (err error) {
|
||||||
|
f.ctl.GetConfig(&f.config)
|
||||||
if err = base.ParseReqExtra[bt.ReqExtra](req); err != nil {
|
if err = base.ParseReqExtra[bt.ReqExtra](req); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -816,10 +817,8 @@ func (f *Fetcher) addTorrent(req *base.Request) (err error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(f.config.Trackers) > 0 {
|
for _, tracker := range f.config.effectiveTrackers() {
|
||||||
for _, tracker := range f.config.Trackers {
|
trackers[tracker] = true
|
||||||
trackers[tracker] = true
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if len(trackers) > 0 {
|
if len(trackers) > 0 {
|
||||||
for tracker := range trackers {
|
for tracker := range trackers {
|
||||||
@@ -947,8 +946,14 @@ func (fm *FetcherManager) AutoRename() bool {
|
|||||||
|
|
||||||
func (fm *FetcherManager) DefaultConfig() any {
|
func (fm *FetcherManager) DefaultConfig() any {
|
||||||
return &config{
|
return &config{
|
||||||
|
TrackerSubscribeURLs: append([]string{}, defaultTrackerSubscribeURLs...),
|
||||||
|
AutoUpdateTrackers: true,
|
||||||
|
CustomTrackers: []string{},
|
||||||
|
ConfigState: fetcher.ConfigState[state]{State: state{
|
||||||
|
TrackerSources: map[string][]string{},
|
||||||
|
TrackerSubscriptionStatuses: map[string]trackerSubscriptionStatus{},
|
||||||
|
}},
|
||||||
ListenPort: 0,
|
ListenPort: 0,
|
||||||
Trackers: []string{},
|
|
||||||
SeedKeep: false,
|
SeedKeep: false,
|
||||||
SeedRatio: 1.0,
|
SeedRatio: 1.0,
|
||||||
SeedTime: 120 * 60,
|
SeedTime: 120 * 60,
|
||||||
|
|||||||
@@ -227,7 +227,7 @@ func buildConfigFetcher(t testing.TB, proxyConfig *base.DownloaderProxyConfig) f
|
|||||||
fetcher := new(FetcherManager).Build()
|
fetcher := new(FetcherManager).Build()
|
||||||
newController := controller.NewController()
|
newController := controller.NewController()
|
||||||
mockCfg := config{
|
mockCfg := config{
|
||||||
Trackers: []string{
|
CustomTrackers: []string{
|
||||||
"udp://tracker.birkenwald.de:6969/announce",
|
"udp://tracker.birkenwald.de:6969/announce",
|
||||||
"udp://tracker.bitsearch.to:1337/announce",
|
"udp://tracker.bitsearch.to:1337/announce",
|
||||||
}}
|
}}
|
||||||
|
|||||||
@@ -0,0 +1,234 @@
|
|||||||
|
package bt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
|
)
|
||||||
|
|
||||||
|
var defaultTrackerSubscribeURLs = []string{
|
||||||
|
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt",
|
||||||
|
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt",
|
||||||
|
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_https.txt",
|
||||||
|
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt",
|
||||||
|
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_udp.txt",
|
||||||
|
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ws.txt",
|
||||||
|
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best.txt",
|
||||||
|
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best_ip.txt",
|
||||||
|
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/all.txt",
|
||||||
|
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best.txt",
|
||||||
|
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/http.txt",
|
||||||
|
}
|
||||||
|
|
||||||
|
func validTracker(raw string) bool {
|
||||||
|
u, err := url.Parse(raw)
|
||||||
|
if err != nil || u.Hostname() == "" || u.User != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch u.Scheme {
|
||||||
|
case "udp", "http", "https", "ws", "wss":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
func uniqueTrackers(groups ...[]string) []string {
|
||||||
|
result := []string{}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, group := range groups {
|
||||||
|
for _, raw := range group {
|
||||||
|
value := strings.TrimSpace(raw)
|
||||||
|
if value != "" && !seen[value] {
|
||||||
|
seen[value] = true
|
||||||
|
result = append(result, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// effectiveTrackers derives the client list without persisting duplicate lists.
|
||||||
|
func (c *config) effectiveTrackers() []string {
|
||||||
|
groups := [][]string{c.CustomTrackers}
|
||||||
|
for _, source := range c.TrackerSubscribeURLs {
|
||||||
|
groups = append(groups, c.State.TrackerSources[source])
|
||||||
|
}
|
||||||
|
return uniqueTrackers(groups...)
|
||||||
|
}
|
||||||
|
func (fm *FetcherManager) NormalizeConfig(raw json.RawMessage) (any, error) {
|
||||||
|
c := fm.DefaultConfig().(*config)
|
||||||
|
if err := json.Unmarshal(raw, c); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if c.ListenPort < 0 || c.ListenPort > 65535 || c.SeedRatio < 0 || c.SeedTime < 0 {
|
||||||
|
return nil, errors.New("invalid BT configuration")
|
||||||
|
}
|
||||||
|
if len(c.TrackerSubscribeURLs) > 64 {
|
||||||
|
return nil, errors.New("too many tracker subscriptions")
|
||||||
|
}
|
||||||
|
c.TrackerSubscribeURLs = uniqueTrackers(c.TrackerSubscribeURLs)
|
||||||
|
for _, source := range c.TrackerSubscribeURLs {
|
||||||
|
u, err := url.Parse(source)
|
||||||
|
if err != nil || u == nil || (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil {
|
||||||
|
return nil, errors.New("invalid tracker subscription URL")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
c.CustomTrackers = uniqueTrackers(c.CustomTrackers)
|
||||||
|
for _, tracker := range c.CustomTrackers {
|
||||||
|
if !validTracker(tracker) {
|
||||||
|
return nil, fmt.Errorf("invalid tracker URL: %s", tracker)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c.State.TrackerSources == nil {
|
||||||
|
c.State.TrackerSources = map[string][]string{}
|
||||||
|
}
|
||||||
|
for source := range c.State.TrackerSources {
|
||||||
|
if !slices.Contains(c.TrackerSubscribeURLs, source) {
|
||||||
|
delete(c.State.TrackerSources, source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c.State.TrackerSubscriptionStatuses == nil {
|
||||||
|
c.State.TrackerSubscriptionStatuses = map[string]trackerSubscriptionStatus{}
|
||||||
|
}
|
||||||
|
for source := range c.State.TrackerSubscriptionStatuses {
|
||||||
|
if !slices.Contains(c.TrackerSubscribeURLs, source) {
|
||||||
|
delete(c.State.TrackerSubscriptionStatuses, source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
func (fm *FetcherManager) CronJobs(host fetcher.ProtocolHost) []fetcher.CronJob {
|
||||||
|
return []fetcher.CronJob{{
|
||||||
|
Name: "update-trackers",
|
||||||
|
Every: 24 * time.Hour,
|
||||||
|
Timeout: 90 * time.Second,
|
||||||
|
Run: func(ctx context.Context, source fetcher.TriggerSource) error {
|
||||||
|
return fm.updateTrackers(ctx, host, source)
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
func parseTrackers(data []byte) ([]string, error) {
|
||||||
|
scanner := bufio.NewScanner(bytes.NewReader(data))
|
||||||
|
scanner.Buffer(make([]byte, 4096), 64*1024)
|
||||||
|
result := []string{}
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := strings.TrimSpace(strings.TrimPrefix(scanner.Text(), "\ufeff"))
|
||||||
|
if validTracker(line) {
|
||||||
|
result = append(result, line)
|
||||||
|
}
|
||||||
|
if len(result) > 20000 {
|
||||||
|
return nil, errors.New("too many trackers")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(result) == 0 {
|
||||||
|
return nil, errors.New("subscription contains no valid trackers")
|
||||||
|
}
|
||||||
|
return uniqueTrackers(result), nil
|
||||||
|
}
|
||||||
|
func (fm *FetcherManager) updateTrackers(ctx context.Context, host fetcher.ProtocolHost, source fetcher.TriggerSource) error {
|
||||||
|
snapshot, err := fetcher.ReadConfig[config](host)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if (source == fetcher.TriggerScheduled && !snapshot.AutoUpdateTrackers) || len(snapshot.TrackerSubscribeURLs) == 0 {
|
||||||
|
return fetcher.ErrJobSkipped
|
||||||
|
}
|
||||||
|
fetched := map[string][]string{}
|
||||||
|
statuses := map[string]trackerSubscriptionStatus{}
|
||||||
|
var failures []error
|
||||||
|
var mu sync.Mutex
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
slots := make(chan struct{}, 4)
|
||||||
|
for _, address := range snapshot.TrackerSubscribeURLs {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(address string) {
|
||||||
|
defer wg.Done()
|
||||||
|
select {
|
||||||
|
case slots <- struct{}{}:
|
||||||
|
defer func() { <-slots }()
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
data, err := host.Fetch(ctx, address, 2<<20, func(body []byte) error { _, err := parseTrackers(body); return err })
|
||||||
|
var trackers []string
|
||||||
|
if err == nil {
|
||||||
|
trackers, err = parseTrackers(data)
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
statuses[address] = trackerSubscriptionStatus{Success: err == nil, TrackerCount: len(trackers), LastFetchTime: time.Now().UTC()}
|
||||||
|
if err != nil {
|
||||||
|
failures = append(failures, fmt.Errorf("subscription %s: %w", address, err))
|
||||||
|
} else {
|
||||||
|
fetched[address] = trackers
|
||||||
|
}
|
||||||
|
}(address)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
failures = append(failures, err)
|
||||||
|
}
|
||||||
|
if len(statuses) == 0 {
|
||||||
|
return errors.Join(failures...)
|
||||||
|
}
|
||||||
|
err = fetcher.UpdateState[config, state](host, func(current *config, state *state) error {
|
||||||
|
if !slices.Equal(current.TrackerSubscribeURLs, snapshot.TrackerSubscribeURLs) {
|
||||||
|
return fetcher.ErrConfigChanged
|
||||||
|
}
|
||||||
|
if state.TrackerSources == nil {
|
||||||
|
state.TrackerSources = map[string][]string{}
|
||||||
|
}
|
||||||
|
if state.TrackerSubscriptionStatuses == nil {
|
||||||
|
state.TrackerSubscriptionStatuses = map[string]trackerSubscriptionStatus{}
|
||||||
|
}
|
||||||
|
for address, status := range statuses {
|
||||||
|
state.TrackerSubscriptionStatuses[address] = status
|
||||||
|
}
|
||||||
|
for address, trackers := range fetched {
|
||||||
|
state.TrackerSources[address] = trackers
|
||||||
|
}
|
||||||
|
if len(failures) == 0 {
|
||||||
|
state.LastTrackerUpdateTime = time.Now().UTC()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
current, err := fetcher.ReadConfig[config](host)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fm.applyTrackers(current.effectiveTrackers())
|
||||||
|
return errors.Join(failures...)
|
||||||
|
}
|
||||||
|
func (fm *FetcherManager) applyTrackers(trackers []string) {
|
||||||
|
lock.Lock()
|
||||||
|
defer lock.Unlock()
|
||||||
|
if client == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
announce := make([][]string, 0, len(trackers))
|
||||||
|
for _, tracker := range trackers {
|
||||||
|
announce = append(announce, []string{tracker})
|
||||||
|
}
|
||||||
|
for _, t := range client.Torrents() {
|
||||||
|
info := t.Info()
|
||||||
|
if info == nil || (info.Private != nil && *info.Private) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
t.AddTrackers(announce)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,217 @@
|
|||||||
|
package bt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
|
)
|
||||||
|
|
||||||
|
func trackerTestHost(t *testing.T, c *config, fetch func(context.Context, string, int64, func([]byte) error) ([]byte, error)) (fetcher.ProtocolHost, *sync.Mutex) {
|
||||||
|
t.Helper()
|
||||||
|
mu := &sync.Mutex{}
|
||||||
|
return fetcher.ProtocolHost{
|
||||||
|
Read: func(dst any) error {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
raw, err := json.Marshal(c)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return json.Unmarshal(raw, dst)
|
||||||
|
},
|
||||||
|
UpdateState: func(change func(config, state json.RawMessage) (any, error)) error {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
raw, _ := json.Marshal(c)
|
||||||
|
currentState, _ := json.Marshal(c.State)
|
||||||
|
next, err := change(raw, currentState)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
raw, err = json.Marshal(next)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return json.Unmarshal(raw, &c.State)
|
||||||
|
},
|
||||||
|
Fetch: fetch,
|
||||||
|
}, mu
|
||||||
|
}
|
||||||
|
func TestTrackerRefreshPartialSuccessUsesLatestConfig(t *testing.T) {
|
||||||
|
fm := &FetcherManager{}
|
||||||
|
cfg := fm.DefaultConfig().(*config)
|
||||||
|
cfg.TrackerSubscribeURLs = []string{"https://a.example/list", "https://b.example/list"}
|
||||||
|
cfg.CustomTrackers = []string{"udp://custom.example:80/announce"}
|
||||||
|
cfg.State.TrackerSources = map[string][]string{cfg.TrackerSubscribeURLs[0]: {"udp://old.example:80/announce"}, cfg.TrackerSubscribeURLs[1]: {"udp://cached.example:80/announce"}}
|
||||||
|
host, mu := trackerTestHost(t, cfg, nil)
|
||||||
|
host.Fetch = func(ctx context.Context, address string, _ int64, validate func([]byte) error) ([]byte, error) {
|
||||||
|
if address == "https://b.example/list" {
|
||||||
|
return nil, errors.New("offline")
|
||||||
|
}
|
||||||
|
// Simulate a frontend update while the network request is in flight.
|
||||||
|
mu.Lock()
|
||||||
|
cfg.CustomTrackers = []string{"udp://new-custom.example:80/announce"}
|
||||||
|
mu.Unlock()
|
||||||
|
data := []byte("udp://fresh.example:80/announce\n# ignored\nudp://fresh.example:80/announce\n")
|
||||||
|
return data, validate(data)
|
||||||
|
}
|
||||||
|
err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("partial failure must be reported")
|
||||||
|
}
|
||||||
|
want := []string{"udp://new-custom.example:80/announce", "udp://fresh.example:80/announce", "udp://cached.example:80/announce"}
|
||||||
|
if !reflect.DeepEqual(cfg.effectiveTrackers(), want) {
|
||||||
|
t.Fatalf("got %v", cfg.effectiveTrackers())
|
||||||
|
}
|
||||||
|
success := cfg.State.TrackerSubscriptionStatuses["https://a.example/list"]
|
||||||
|
failure := cfg.State.TrackerSubscriptionStatuses["https://b.example/list"]
|
||||||
|
if !success.Success || success.TrackerCount != 1 || success.LastFetchTime.IsZero() || failure.Success || failure.TrackerCount != 0 || failure.LastFetchTime.IsZero() {
|
||||||
|
t.Fatalf("unexpected subscription statuses: %+v", cfg.State.TrackerSubscriptionStatuses)
|
||||||
|
}
|
||||||
|
if !cfg.State.LastTrackerUpdateTime.IsZero() {
|
||||||
|
t.Fatal("partial refresh marked fully successful")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestTrackerRefreshDiscardsChangedSubscriptions(t *testing.T) {
|
||||||
|
fm := &FetcherManager{}
|
||||||
|
cfg := fm.DefaultConfig().(*config)
|
||||||
|
cfg.TrackerSubscribeURLs = []string{"https://a.example/list"}
|
||||||
|
host, mu := trackerTestHost(t, cfg, nil)
|
||||||
|
host.Fetch = func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
|
||||||
|
mu.Lock()
|
||||||
|
cfg.TrackerSubscribeURLs = []string{}
|
||||||
|
mu.Unlock()
|
||||||
|
return []byte("udp://fresh.example:80/announce"), nil
|
||||||
|
}
|
||||||
|
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); !errors.Is(err, fetcher.ErrConfigChanged) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(cfg.State.TrackerSources) != 0 {
|
||||||
|
t.Fatal("outdated source was committed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestTrackerRefreshManualWorksWhenAutomaticDisabled(t *testing.T) {
|
||||||
|
fm := &FetcherManager{}
|
||||||
|
cfg := fm.DefaultConfig().(*config)
|
||||||
|
cfg.AutoUpdateTrackers = false
|
||||||
|
cfg.TrackerSubscribeURLs = []string{"https://a.example/list"}
|
||||||
|
calls := 0
|
||||||
|
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
|
||||||
|
calls++
|
||||||
|
return []byte("udp://fresh.example:80/announce"), nil
|
||||||
|
})
|
||||||
|
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerScheduled); !errors.Is(err, fetcher.ErrJobSkipped) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if calls != 0 {
|
||||||
|
t.Fatal("scheduled job fetched while disabled")
|
||||||
|
}
|
||||||
|
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if calls != 1 || cfg.State.LastTrackerUpdateTime.IsZero() {
|
||||||
|
t.Fatal("manual refresh did not succeed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestTrackerConfigPreservesExplicitSettings(t *testing.T) {
|
||||||
|
value, err := (&FetcherManager{}).NormalizeConfig(json.RawMessage(`{
|
||||||
|
"listenPort":6881, "trackerSubscribeUrls":[], "autoUpdateTrackers":false,
|
||||||
|
"customTrackers":["udp://custom.example:80/announce"]
|
||||||
|
}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg := value.(*config)
|
||||||
|
if cfg.AutoUpdateTrackers || len(cfg.TrackerSubscribeURLs) != 0 || cfg.ListenPort != 6881 {
|
||||||
|
t.Fatalf("%+v", cfg)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(cfg.effectiveTrackers(), cfg.CustomTrackers) {
|
||||||
|
t.Fatalf("derived trackers: %v", cfg.effectiveTrackers())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseTrackerList(t *testing.T) {
|
||||||
|
got, err := parseTrackers([]byte("\ufeffudp://tracker.example:80/announce\r\n\n<html>error</html>\nhttps://tracker.example/announce\n"))
|
||||||
|
if err != nil || len(got) != 2 {
|
||||||
|
t.Fatalf("%v %v", got, err)
|
||||||
|
}
|
||||||
|
if _, err := parseTrackers([]byte("<html>mirror error</html>")); err == nil {
|
||||||
|
t.Fatal("invalid mirror body accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestTrackerDefaultsRegisterDailyJob(t *testing.T) {
|
||||||
|
jobs := (&FetcherManager{}).CronJobs(fetcher.ProtocolHost{})
|
||||||
|
if len(jobs) != 1 || jobs[0].Name != "update-trackers" || jobs[0].Every != 24*time.Hour {
|
||||||
|
t.Fatal(jobs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrackerRefreshAllFailuresPersistStatusesAndKeepCache(t *testing.T) {
|
||||||
|
fm := &FetcherManager{}
|
||||||
|
cfg := fm.DefaultConfig().(*config)
|
||||||
|
address := "https://custom.example/trackers.txt"
|
||||||
|
cfg.TrackerSubscribeURLs = []string{address}
|
||||||
|
cfg.State.TrackerSources[address] = []string{"udp://cached.example:80/announce"}
|
||||||
|
cfg.State.TrackerSubscriptionStatuses[address] = trackerSubscriptionStatus{Success: true, TrackerCount: 1, LastFetchTime: time.Now().Add(-time.Hour)}
|
||||||
|
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
|
||||||
|
return nil, errors.New("offline")
|
||||||
|
})
|
||||||
|
before := time.Now()
|
||||||
|
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); err == nil {
|
||||||
|
t.Fatal("failure was not reported")
|
||||||
|
}
|
||||||
|
status, ok := cfg.State.TrackerSubscriptionStatuses[address]
|
||||||
|
if !ok || status.Success || status.TrackerCount != 0 || status.LastFetchTime.Before(before) {
|
||||||
|
t.Fatalf("failed fetch status not stored: %+v", status)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(cfg.State.TrackerSources[address], []string{"udp://cached.example:80/announce"}) {
|
||||||
|
t.Fatal("last good cache lost")
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(cfg.effectiveTrackers(), cfg.State.TrackerSources[address]) {
|
||||||
|
t.Fatal("cached trackers not retained")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrackerConfigPrunesRemovedSubscriptions(t *testing.T) {
|
||||||
|
fm := &FetcherManager{}
|
||||||
|
cfg := fm.DefaultConfig().(*config)
|
||||||
|
address := "https://custom.example/trackers.txt"
|
||||||
|
cfg.TrackerSubscribeURLs = []string{}
|
||||||
|
cfg.State.TrackerSources[address] = []string{"udp://cached.example:80/announce"}
|
||||||
|
cfg.State.TrackerSubscriptionStatuses[address] = trackerSubscriptionStatus{Success: true, TrackerCount: 1, LastFetchTime: time.Now()}
|
||||||
|
raw, _ := json.Marshal(cfg)
|
||||||
|
value, err := fm.NormalizeConfig(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
next := value.(*config)
|
||||||
|
if len(next.State.TrackerSources) != 0 || len(next.State.TrackerSubscriptionStatuses) != 0 || len(next.effectiveTrackers()) != 0 {
|
||||||
|
t.Fatal("removed subscription still owns cached results")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrackerRefreshTimeoutPersistsCompletedAttemptStatus(t *testing.T) {
|
||||||
|
fm := &FetcherManager{}
|
||||||
|
cfg := fm.DefaultConfig().(*config)
|
||||||
|
address := "https://custom.example/trackers.txt"
|
||||||
|
cfg.TrackerSubscribeURLs = []string{address}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
|
||||||
|
cancel()
|
||||||
|
return nil, context.DeadlineExceeded
|
||||||
|
})
|
||||||
|
if err := fm.updateTrackers(ctx, host, fetcher.TriggerManual); !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
status, ok := cfg.State.TrackerSubscriptionStatuses[address]
|
||||||
|
if !ok || status.Success || status.TrackerCount != 0 || status.LastFetchTime.IsZero() {
|
||||||
|
t.Fatalf("timeout status lost: %+v", status)
|
||||||
|
}
|
||||||
|
}
|
||||||
+6
-1
@@ -2,6 +2,7 @@ package api
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/url"
|
||||||
pathpkg "path"
|
pathpkg "path"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
@@ -75,7 +76,11 @@ func (r RouteSpec) Match(rawPath string) (map[string]string, bool) {
|
|||||||
params := make(map[string]string)
|
params := make(map[string]string)
|
||||||
for index, part := range r.parts {
|
for index, part := range r.parts {
|
||||||
if part.param != "" {
|
if part.param != "" {
|
||||||
params[part.param] = segments[index]
|
value, err := url.PathUnescape(segments[index])
|
||||||
|
if err != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
params[part.param] = value
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if part.literal != segments[index] {
|
if part.literal != segments[index] {
|
||||||
|
|||||||
@@ -2,7 +2,9 @@ package api
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
@@ -62,6 +64,8 @@ func NewService(downloader *download.Downloader) (*Service, error) {
|
|||||||
newRoute(http.MethodGet, "/api/v1/tasks/{id}/status", service.getTaskStatus),
|
newRoute(http.MethodGet, "/api/v1/tasks/{id}/status", service.getTaskStatus),
|
||||||
newRoute(http.MethodGet, "/api/v1/tasks", service.getTasks),
|
newRoute(http.MethodGet, "/api/v1/tasks", service.getTasks),
|
||||||
newRoute(http.MethodGet, "/api/v1/tasks/{id}/stats", service.getStats),
|
newRoute(http.MethodGet, "/api/v1/tasks/{id}/stats", service.getStats),
|
||||||
|
newRoute(http.MethodPost, "/api/v1/protocols/{protocol}/jobs/{name}/trigger", service.triggerProtocolJob),
|
||||||
|
newRoute(http.MethodPost, "/api/v1/mirrors/{provider}/candidates", service.mirrorCandidates),
|
||||||
newRoute(http.MethodGet, "/api/v1/config", service.getConfig),
|
newRoute(http.MethodGet, "/api/v1/config", service.getConfig),
|
||||||
newRoute(http.MethodPut, "/api/v1/config", service.putConfig),
|
newRoute(http.MethodPut, "/api/v1/config", service.putConfig),
|
||||||
newRoute(http.MethodPost, "/api/v1/extensions", service.installExtension),
|
newRoute(http.MethodPost, "/api/v1/extensions", service.installExtension),
|
||||||
@@ -294,6 +298,36 @@ func (s *Service) getStats(ctx *Context) *Response {
|
|||||||
return newOKResponse(stats)
|
return newOKResponse(stats)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Service) triggerProtocolJob(ctx *Context) *Response {
|
||||||
|
if err := s.Downloader.TriggerProtocolJob(context.Background(), ctx.PathParam("protocol"), ctx.PathParam("name")); err != nil {
|
||||||
|
status := http.StatusBadRequest
|
||||||
|
if errors.Is(err, download.ErrUnSupportedProtocol) || errors.Is(err, download.ErrProtocolJobNotFound) {
|
||||||
|
status = http.StatusNotFound
|
||||||
|
}
|
||||||
|
return newStatusResponse(status, restmodel.NewErrorResult(err.Error()))
|
||||||
|
}
|
||||||
|
return newNilResponse()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) mirrorCandidates(ctx *Context) *Response {
|
||||||
|
var request restmodel.MirrorCandidatesReq
|
||||||
|
if response := decodeRequest(ctx.Body, &request); response != nil {
|
||||||
|
return response
|
||||||
|
}
|
||||||
|
urls, err := s.Downloader.GetMirrorCandidates(ctx.PathParam("provider"), request.URL)
|
||||||
|
if err != nil {
|
||||||
|
status := http.StatusInternalServerError
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, download.ErrMirrorProviderNotFound):
|
||||||
|
status = http.StatusNotFound
|
||||||
|
case errors.Is(err, download.ErrInvalidMirrorURL):
|
||||||
|
status = http.StatusBadRequest
|
||||||
|
}
|
||||||
|
return newStatusResponse(status, restmodel.NewErrorResult(err.Error()))
|
||||||
|
}
|
||||||
|
return newOKResponse(&restmodel.MirrorCandidatesResp{URLs: urls})
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Service) getConfig(_ *Context) *Response {
|
func (s *Service) getConfig(_ *Context) *Response {
|
||||||
config, err := s.Downloader.GetConfig()
|
config, err := s.Downloader.GetConfig()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package api
|
package api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
@@ -8,7 +9,10 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
|
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
"github.com/GopeedLab/gopeed/pkg/download"
|
"github.com/GopeedLab/gopeed/pkg/download"
|
||||||
restmodel "github.com/GopeedLab/gopeed/pkg/rest/model"
|
restmodel "github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||||
@@ -80,3 +84,186 @@ func serviceTestRequest[T any](t *testing.T, service *Service, method, path stri
|
|||||||
}
|
}
|
||||||
return result.Data
|
return result.Data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFullConfigDispatch(t *testing.T) {
|
||||||
|
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&httpProtocol.FetcherManager{}}})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
service, err := NewService(d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
response := service.Dispatch(http.MethodPut, "/api/v1/config", nil, []byte(`{"protocolConfig":{"http":{"connections":3,"useServerCtime":false}}}`), nil)
|
||||||
|
if response.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("%+v", response)
|
||||||
|
}
|
||||||
|
status, body := service.JSON(http.MethodGet, "/api/v1/config", nil, nil, nil)
|
||||||
|
var result struct {
|
||||||
|
Data struct {
|
||||||
|
ProtocolConfig map[string]struct {
|
||||||
|
Connections int `json:"connections"`
|
||||||
|
} `json:"protocolConfig"`
|
||||||
|
} `json:"data"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &result); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if status != http.StatusOK || result.Data.ProtocolConfig["http"].Connections != 3 {
|
||||||
|
t.Fatalf("%d %s", status, body)
|
||||||
|
}
|
||||||
|
if response := service.Dispatch(http.MethodPost, "/api/v1/protocols/http/jobs/missing/trigger", nil, nil, nil); response.StatusCode != http.StatusNotFound {
|
||||||
|
t.Fatalf("%+v", response)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativePathParametersAreDecodedOnce(t *testing.T) {
|
||||||
|
route := newRoute(http.MethodGet, "/api/v1/tasks/{id}/status", func(c *Context) *Response { return newOKResponse(c.PathParam("id")) })
|
||||||
|
for _, test := range []struct{ path, want string }{
|
||||||
|
{"/api/v1/tasks/%E5%BC%80%E5%8F%91%E8%80%85%40name/status", "开发者@name"},
|
||||||
|
{"/api/v1/tasks/literal%252Fname/status", "literal%2Fname"},
|
||||||
|
} {
|
||||||
|
response := (&Service{routes: []RouteSpec{route}}).Dispatch(http.MethodGet, test.path, nil, nil, nil)
|
||||||
|
raw, _ := json.Marshal(response.Body)
|
||||||
|
var body struct {
|
||||||
|
Data string `json:"data"`
|
||||||
|
}
|
||||||
|
_ = json.Unmarshal(raw, &body)
|
||||||
|
if body.Data != test.want {
|
||||||
|
t.Fatalf("%s => %q", test.path, body.Data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMirrorCandidates(t *testing.T) {
|
||||||
|
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&httpProtocol.FetcherManager{}}})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
cfg, _ := d.GetConfig()
|
||||||
|
cfg.GithubMirror = &base.GithubMirrorConfig{Enabled: true, Mirrors: []base.GithubMirror{
|
||||||
|
{Type: "jsdelivr", URL: "https://cdn.example/gh"},
|
||||||
|
{Type: "ghProxy", URL: "https://proxy.example"},
|
||||||
|
{Type: "ghProxy", URL: "https://proxy.example"},
|
||||||
|
}}
|
||||||
|
if err := d.PutConfig(cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
service, err := NewService(d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, official := range []string{
|
||||||
|
"https://github.com/GopeedLab/gopeed/releases/download/v2.0.0/package.zip",
|
||||||
|
"https://raw.githubusercontent.com/owner/repo/main/trackers.txt",
|
||||||
|
} {
|
||||||
|
request, _ := json.Marshal(map[string]string{"url": official})
|
||||||
|
status, raw := service.JSON(http.MethodPost, "/api/v1/mirrors/github/candidates", nil, request, nil)
|
||||||
|
var body struct {
|
||||||
|
Data struct {
|
||||||
|
URLs []string `json:"urls"`
|
||||||
|
} `json:"data"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := 2
|
||||||
|
if official == "https://raw.githubusercontent.com/owner/repo/main/trackers.txt" {
|
||||||
|
want = 3
|
||||||
|
}
|
||||||
|
urls := body.Data.URLs
|
||||||
|
if status != http.StatusOK || len(urls) != want || urls[0] != official || urls[len(urls)-1] != "https://proxy.example/"+official {
|
||||||
|
t.Fatalf("%d %s", status, raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, invalid := range []string{"https://api.github.com/repos/GopeedLab/gopeed/releases?per_page=10", "", "https://example.com/x", "http://github.com/x", "https://github.com.evil.test/x", "https://[email protected]/x", "https://github.com:443/x", "https://github.com:/x", "https://github.com/x#fragment", "https://github.com/x#", "/relative", "https://github.com%2fevil.test/x"} {
|
||||||
|
request, _ := json.Marshal(map[string]string{"url": invalid})
|
||||||
|
if response := service.Dispatch(http.MethodPost, "/api/v1/mirrors/github/candidates", nil, request, nil); response.StatusCode != http.StatusBadRequest {
|
||||||
|
t.Fatalf("accepted %s: %+v", invalid, response)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cfg.GithubMirror.Enabled = false
|
||||||
|
if err := d.PutConfig(cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
status, raw := service.JSON(http.MethodPost, "/api/v1/mirrors/github/candidates", nil, []byte(`{"url":"https://github.com/owner/repo"}`), nil)
|
||||||
|
var body struct {
|
||||||
|
Data struct {
|
||||||
|
URLs []string `json:"urls"`
|
||||||
|
} `json:"data"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &body); err != nil || status != http.StatusOK || len(body.Data.URLs) != 1 || body.Data.URLs[0] != "https://github.com/owner/repo" {
|
||||||
|
t.Fatalf("%d %s", status, raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, provider := range []string{"unknown", "GitHub"} {
|
||||||
|
if response := service.Dispatch(http.MethodPost, "/api/v1/mirrors/"+provider+"/candidates", nil, []byte(`{"url":"https://github.com/owner/repo"}`), nil); response.StatusCode != http.StatusNotFound {
|
||||||
|
t.Fatalf("unknown provider accepted: %+v", response)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type apiJobState struct {
|
||||||
|
Runs int `json:"runs"`
|
||||||
|
}
|
||||||
|
type apiJobConfig struct {
|
||||||
|
fetcher.ConfigState[apiJobState]
|
||||||
|
Writable int `json:"writable"`
|
||||||
|
}
|
||||||
|
type apiStateJobManager struct{ httpProtocol.FetcherManager }
|
||||||
|
|
||||||
|
func (*apiStateJobManager) Name() string { return "state-api" }
|
||||||
|
func (*apiStateJobManager) DefaultConfig() any { return &apiJobConfig{Writable: 7} }
|
||||||
|
func (*apiStateJobManager) CronJobs(host fetcher.ProtocolHost) []fetcher.CronJob {
|
||||||
|
return []fetcher.CronJob{{Name: "refresh", Every: time.Hour, Timeout: time.Second, Run: func(_ context.Context, source fetcher.TriggerSource) error {
|
||||||
|
if source == fetcher.TriggerScheduled {
|
||||||
|
return fetcher.ErrJobSkipped
|
||||||
|
}
|
||||||
|
return fetcher.UpdateState[apiJobConfig, apiJobState](host, func(cfg *apiJobConfig, state *apiJobState) error { cfg.Writable = 999; state.Runs++; return nil })
|
||||||
|
}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFullConfigAndJobTriggerShareProtectedProtocolState(t *testing.T) {
|
||||||
|
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&apiStateJobManager{}}})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
service, err := NewService(d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
put := []byte(`{"protocolConfig":{"state-api":{"writable":9,"state":"ignore forged state"}}}`)
|
||||||
|
if response := service.Dispatch(http.MethodPut, "/api/v1/config", nil, put, nil); response.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("%+v", response)
|
||||||
|
}
|
||||||
|
if response := service.Dispatch(http.MethodPost, "/api/v1/protocols/state-api/jobs/refresh/trigger", nil, nil, nil); response.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("%+v", response)
|
||||||
|
}
|
||||||
|
status, raw := service.JSON(http.MethodGet, "/api/v1/config", nil, nil, nil)
|
||||||
|
var response struct {
|
||||||
|
Data struct {
|
||||||
|
ProtocolConfig map[string]apiJobConfig `json:"protocolConfig"`
|
||||||
|
} `json:"data"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &response); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg := response.Data.ProtocolConfig["state-api"]
|
||||||
|
if status != http.StatusOK || cfg.Writable != 9 || cfg.State.Runs != 1 {
|
||||||
|
t.Fatalf("%d %s", status, raw)
|
||||||
|
}
|
||||||
|
// Replaying an old full PUT must not undo the manual job's state change.
|
||||||
|
if response := service.Dispatch(http.MethodPut, "/api/v1/config", nil, put, nil); response.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("%+v", response)
|
||||||
|
}
|
||||||
|
_, raw = service.JSON(http.MethodGet, "/api/v1/config", nil, nil, nil)
|
||||||
|
if err := json.Unmarshal(raw, &response); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if response.Data.ProtocolConfig["state-api"].State.Runs != 1 {
|
||||||
|
t.Fatalf("job state overwritten: %s", raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
package base
|
||||||
|
|
||||||
|
import "github.com/GopeedLab/gopeed/internal/githubmirror"
|
||||||
|
|
||||||
|
// Preserve downloader configuration type names and the existing JSON schema.
|
||||||
|
type GithubMirror = githubmirror.Mirror
|
||||||
|
type GithubMirrorConfig = githubmirror.Config
|
||||||
|
|
||||||
|
func DefaultGithubMirrorConfig() *GithubMirrorConfig {
|
||||||
|
return githubmirror.DefaultConfig()
|
||||||
|
}
|
||||||
@@ -181,6 +181,7 @@ type DownloaderStoreConfig struct {
|
|||||||
DownloadDir string `json:"downloadDir"` // DownloadDir is the default directory to save the downloaded files
|
DownloadDir string `json:"downloadDir"` // DownloadDir is the default directory to save the downloaded files
|
||||||
MaxRunning int `json:"maxRunning"` // MaxRunning is the max running download count
|
MaxRunning int `json:"maxRunning"` // MaxRunning is the max running download count
|
||||||
ProtocolConfig map[string]any `json:"protocolConfig"` // ProtocolConfig is special config for each protocol
|
ProtocolConfig map[string]any `json:"protocolConfig"` // ProtocolConfig is special config for each protocol
|
||||||
|
GithubMirror *GithubMirrorConfig `json:"githubMirror"`
|
||||||
Extra map[string]any `json:"extra"`
|
Extra map[string]any `json:"extra"`
|
||||||
Proxy *DownloaderProxyConfig `json:"proxy"`
|
Proxy *DownloaderProxyConfig `json:"proxy"`
|
||||||
Webhook *WebhookConfig `json:"webhook"` // Webhook is the webhook configuration
|
Webhook *WebhookConfig `json:"webhook"` // Webhook is the webhook configuration
|
||||||
@@ -199,6 +200,9 @@ func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
|
|||||||
if cfg.ProtocolConfig == nil {
|
if cfg.ProtocolConfig == nil {
|
||||||
cfg.ProtocolConfig = make(map[string]any)
|
cfg.ProtocolConfig = make(map[string]any)
|
||||||
}
|
}
|
||||||
|
if cfg.GithubMirror == nil {
|
||||||
|
cfg.GithubMirror = DefaultGithubMirrorConfig()
|
||||||
|
}
|
||||||
if cfg.Proxy == nil {
|
if cfg.Proxy == nil {
|
||||||
cfg.Proxy = &DownloaderProxyConfig{}
|
cfg.Proxy = &DownloaderProxyConfig{}
|
||||||
}
|
}
|
||||||
@@ -239,6 +243,9 @@ func (cfg *DownloaderStoreConfig) Merge(beforeCfg *DownloaderStoreConfig) *Downl
|
|||||||
if cfg.Extra == nil {
|
if cfg.Extra == nil {
|
||||||
cfg.Extra = beforeCfg.Extra
|
cfg.Extra = beforeCfg.Extra
|
||||||
}
|
}
|
||||||
|
if cfg.GithubMirror == nil {
|
||||||
|
cfg.GithubMirror = beforeCfg.GithubMirror
|
||||||
|
}
|
||||||
if cfg.Proxy == nil {
|
if cfg.Proxy == nil {
|
||||||
cfg.Proxy = beforeCfg.Proxy
|
cfg.Proxy = beforeCfg.Proxy
|
||||||
}
|
}
|
||||||
|
|||||||
+22
-1
@@ -15,6 +15,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
|||||||
"Init",
|
"Init",
|
||||||
&DownloaderStoreConfig{},
|
&DownloaderStoreConfig{},
|
||||||
&DownloaderStoreConfig{
|
&DownloaderStoreConfig{
|
||||||
|
GithubMirror: DefaultGithubMirrorConfig(),
|
||||||
MaxRunning: 5,
|
MaxRunning: 5,
|
||||||
ProtocolConfig: map[string]any{},
|
ProtocolConfig: map[string]any{},
|
||||||
Proxy: &DownloaderProxyConfig{},
|
Proxy: &DownloaderProxyConfig{},
|
||||||
@@ -36,6 +37,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
|||||||
MaxRunning: 10,
|
MaxRunning: 10,
|
||||||
},
|
},
|
||||||
&DownloaderStoreConfig{
|
&DownloaderStoreConfig{
|
||||||
|
GithubMirror: DefaultGithubMirrorConfig(),
|
||||||
MaxRunning: 10,
|
MaxRunning: 10,
|
||||||
ProtocolConfig: map[string]any{},
|
ProtocolConfig: map[string]any{},
|
||||||
Proxy: &DownloaderProxyConfig{},
|
Proxy: &DownloaderProxyConfig{},
|
||||||
@@ -59,7 +61,8 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
&DownloaderStoreConfig{
|
&DownloaderStoreConfig{
|
||||||
MaxRunning: 5,
|
GithubMirror: DefaultGithubMirrorConfig(),
|
||||||
|
MaxRunning: 5,
|
||||||
ProtocolConfig: map[string]any{
|
ProtocolConfig: map[string]any{
|
||||||
"key": "value",
|
"key": "value",
|
||||||
},
|
},
|
||||||
@@ -84,6 +87,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
&DownloaderStoreConfig{
|
&DownloaderStoreConfig{
|
||||||
|
GithubMirror: DefaultGithubMirrorConfig(),
|
||||||
MaxRunning: 5,
|
MaxRunning: 5,
|
||||||
ProtocolConfig: map[string]any{},
|
ProtocolConfig: map[string]any{},
|
||||||
Proxy: &DownloaderProxyConfig{
|
Proxy: &DownloaderProxyConfig{
|
||||||
@@ -110,6 +114,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
&DownloaderStoreConfig{
|
&DownloaderStoreConfig{
|
||||||
|
GithubMirror: DefaultGithubMirrorConfig(),
|
||||||
MaxRunning: 5,
|
MaxRunning: 5,
|
||||||
ProtocolConfig: map[string]any{},
|
ProtocolConfig: map[string]any{},
|
||||||
Proxy: &DownloaderProxyConfig{},
|
Proxy: &DownloaderProxyConfig{},
|
||||||
@@ -134,6 +139,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
&DownloaderStoreConfig{
|
&DownloaderStoreConfig{
|
||||||
|
GithubMirror: DefaultGithubMirrorConfig(),
|
||||||
MaxRunning: 5,
|
MaxRunning: 5,
|
||||||
ProtocolConfig: map[string]any{},
|
ProtocolConfig: map[string]any{},
|
||||||
Proxy: &DownloaderProxyConfig{},
|
Proxy: &DownloaderProxyConfig{},
|
||||||
@@ -149,6 +155,20 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "Init explicitly disabled mirrors with empty list",
|
||||||
|
fields: &DownloaderStoreConfig{GithubMirror: &GithubMirrorConfig{Enabled: false, Mirrors: []GithubMirror{}}},
|
||||||
|
want: &DownloaderStoreConfig{
|
||||||
|
MaxRunning: 5,
|
||||||
|
ProtocolConfig: map[string]any{},
|
||||||
|
GithubMirror: &GithubMirrorConfig{Enabled: false, Mirrors: []GithubMirror{}},
|
||||||
|
Proxy: &DownloaderProxyConfig{},
|
||||||
|
Webhook: &WebhookConfig{},
|
||||||
|
Script: &ScriptConfig{},
|
||||||
|
AutoTorrent: &AutoTorrentConfig{},
|
||||||
|
Archive: &ArchiveConfig{},
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
@@ -157,6 +177,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
|||||||
DownloadDir: tt.fields.DownloadDir,
|
DownloadDir: tt.fields.DownloadDir,
|
||||||
MaxRunning: tt.fields.MaxRunning,
|
MaxRunning: tt.fields.MaxRunning,
|
||||||
ProtocolConfig: tt.fields.ProtocolConfig,
|
ProtocolConfig: tt.fields.ProtocolConfig,
|
||||||
|
GithubMirror: tt.fields.GithubMirror,
|
||||||
Extra: tt.fields.Extra,
|
Extra: tt.fields.Extra,
|
||||||
Proxy: tt.fields.Proxy,
|
Proxy: tt.fields.Proxy,
|
||||||
Webhook: tt.fields.Webhook,
|
Webhook: tt.fields.Webhook,
|
||||||
|
|||||||
@@ -0,0 +1,245 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
|
btProtocol "github.com/GopeedLab/gopeed/internal/protocol/bt"
|
||||||
|
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
|
)
|
||||||
|
|
||||||
|
type testJobManager struct {
|
||||||
|
httpProtocol.FetcherManager
|
||||||
|
job fetcher.CronJob
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *testJobManager) Name() string { return "test" }
|
||||||
|
func (m *testJobManager) CronJobs(fetcher.ProtocolHost) []fetcher.CronJob {
|
||||||
|
return []fetcher.CronJob{m.job}
|
||||||
|
}
|
||||||
|
func testBackendDownloader(t *testing.T, managers ...fetcher.FetcherManager) *Downloader {
|
||||||
|
t.Helper()
|
||||||
|
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: managers})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := d.Close(); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
func TestProtocolJobCoalescesAndCancelsOnClose(t *testing.T) {
|
||||||
|
entered := make(chan struct{})
|
||||||
|
var calls atomic.Int32
|
||||||
|
manager := &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Minute, Run: func(ctx context.Context, _ fetcher.TriggerSource) error {
|
||||||
|
if calls.Add(1) == 1 {
|
||||||
|
close(entered)
|
||||||
|
}
|
||||||
|
<-ctx.Done()
|
||||||
|
return ctx.Err()
|
||||||
|
}}}
|
||||||
|
d := testBackendDownloader(t, manager)
|
||||||
|
result := make(chan error, 2)
|
||||||
|
go func() { result <- d.TriggerProtocolJob(context.Background(), "test", "refresh") }()
|
||||||
|
<-entered
|
||||||
|
waiterCtx, cancel := context.WithCancel(context.Background())
|
||||||
|
// A canceled waiter joins the existing work without canceling it.
|
||||||
|
cancel()
|
||||||
|
if err := d.TriggerProtocolJob(waiterCtx, "test", "refresh"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if calls.Load() != 1 {
|
||||||
|
t.Fatalf("executed %d times", calls.Load())
|
||||||
|
}
|
||||||
|
if err := d.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := <-result; !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := d.TriggerProtocolJob(context.Background(), "test", "missing"); !errors.Is(err, ErrProtocolJobNotFound) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestProtocolJobPersistsSuccess(t *testing.T) {
|
||||||
|
manager := &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Minute, Run: func(context.Context, fetcher.TriggerSource) error { return nil }}}
|
||||||
|
d := testBackendDownloader(t, manager)
|
||||||
|
if err := d.TriggerProtocolJob(context.Background(), "test", "refresh"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var state protocolJobState
|
||||||
|
exists, err := d.storage.Get(bucketProtocolJobs, "test/refresh", &state)
|
||||||
|
if err != nil || !exists || state.LastSuccess.IsZero() {
|
||||||
|
t.Fatalf("state %+v: %v", state, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestRaceResourceValidatesResponsesAndCancelsLosers(t *testing.T) {
|
||||||
|
loserCanceled := make(chan struct{})
|
||||||
|
loserStarted := make(chan struct{})
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch {
|
||||||
|
case r.Method == http.MethodConnect:
|
||||||
|
http.Error(w, "official unavailable", http.StatusBadGateway)
|
||||||
|
case strings.HasPrefix(r.URL.Path, "/bad/"):
|
||||||
|
io.WriteString(w, "<html>proxy landing page</html>")
|
||||||
|
case strings.HasPrefix(r.URL.Path, "/good/"):
|
||||||
|
<-loserStarted
|
||||||
|
io.WriteString(w, `{"ok":true}`)
|
||||||
|
case strings.HasPrefix(r.URL.Path, "/slow/"):
|
||||||
|
close(loserStarted)
|
||||||
|
<-r.Context().Done()
|
||||||
|
close(loserCanceled)
|
||||||
|
default:
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
d := testBackendDownloader(t, &httpProtocol.FetcherManager{})
|
||||||
|
cfg, _ := d.GetConfig()
|
||||||
|
cfg.Proxy = &base.DownloaderProxyConfig{Enable: true, Scheme: "http", Host: strings.TrimPrefix(server.URL, "http://")}
|
||||||
|
cfg.GithubMirror = &base.GithubMirrorConfig{Enabled: true, Mirrors: []base.GithubMirror{
|
||||||
|
{Type: "ghProxy", URL: server.URL + "/bad"},
|
||||||
|
{Type: "ghProxy", URL: server.URL + "/slow"},
|
||||||
|
{Type: "ghProxy", URL: server.URL + "/good"},
|
||||||
|
}}
|
||||||
|
if err := d.PutConfig(cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := d.fetchResource(context.Background(), "https://raw.githubusercontent.com/owner/repo/main/test.json", 1024, false, func(body []byte) error {
|
||||||
|
if !json.Valid(body) {
|
||||||
|
return errors.New("invalid JSON")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil || string(body) != `{"ok":true}` {
|
||||||
|
t.Fatalf("%s %v", body, err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-loserCanceled:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("losing request was not canceled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProtocolConfigInitializesCurrentSchema(t *testing.T) {
|
||||||
|
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&httpProtocol.FetcherManager{}}})
|
||||||
|
d.cfg.DownloaderStoreConfig = &base.DownloaderStoreConfig{
|
||||||
|
GithubMirror: &base.GithubMirrorConfig{Enabled: false, Mirrors: []base.GithubMirror{}},
|
||||||
|
ProtocolConfig: map[string]any{"http": map[string]any{"connections": 3}},
|
||||||
|
}
|
||||||
|
if err := d.storage.Setup([]string{bucketConfig}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.storage.Close()
|
||||||
|
if err := d.initProtocolConfigs(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if d.cfg.GithubMirror.Enabled || len(d.cfg.GithubMirror.Mirrors) != 0 {
|
||||||
|
t.Fatal("explicit disabled/empty mirrors lost")
|
||||||
|
}
|
||||||
|
value, err := d.readProtocolConfig("http")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(value)
|
||||||
|
var cfg struct {
|
||||||
|
Connections int `json:"connections"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &cfg); err != nil || cfg.Connections != 3 {
|
||||||
|
t.Fatalf("current protocol config: %s %v", raw, err)
|
||||||
|
}
|
||||||
|
if err := d.initProtocolConfigs(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProtocolConfigDoesNotImportOldExtraSettings(t *testing.T) {
|
||||||
|
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&btProtocol.FetcherManager{}}})
|
||||||
|
d.cfg.DownloaderStoreConfig = &base.DownloaderStoreConfig{
|
||||||
|
Extra: map[string]any{
|
||||||
|
"githubMirror": "obsolete invalid value",
|
||||||
|
"bt": map[string]any{"customTrackers": []string{"udp://old.example:80/announce"}, "autoUpdateTrackers": true},
|
||||||
|
},
|
||||||
|
ProtocolConfig: map[string]any{"bt": map[string]any{"autoUpdateTrackers": false, "listenPort": 6881}},
|
||||||
|
}
|
||||||
|
if err := d.storage.Setup([]string{bucketConfig}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.storage.Close()
|
||||||
|
if err := d.initProtocolConfigs(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !d.cfg.GithubMirror.Enabled || len(d.cfg.GithubMirror.Mirrors) != 4 {
|
||||||
|
t.Fatal("missing root mirror config must use new defaults")
|
||||||
|
}
|
||||||
|
value, err := d.readProtocolConfig("bt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(value)
|
||||||
|
var cfg struct {
|
||||||
|
ListenPort int `json:"listenPort"`
|
||||||
|
AutoUpdateTrackers bool `json:"autoUpdateTrackers"`
|
||||||
|
CustomTrackers []string `json:"customTrackers"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &cfg); err != nil || cfg.ListenPort != 6881 || cfg.AutoUpdateTrackers || len(cfg.CustomTrackers) != 0 {
|
||||||
|
t.Fatalf("old extra settings affected protocol config: %s %v", raw, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProtocolJobsRunAutomaticallyAndRestoreSuccess(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
executed := make(chan fetcher.TriggerSource, 4)
|
||||||
|
manager := func() *testJobManager {
|
||||||
|
return &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Second, Run: func(_ context.Context, source fetcher.TriggerSource) error { executed <- source; return nil }}}
|
||||||
|
}
|
||||||
|
makeDownloader := func() *Downloader {
|
||||||
|
d := NewDownloader(&DownloaderConfig{Storage: NewBoltStorage(dir), StorageDir: dir, TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{manager()}})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
d := makeDownloader()
|
||||||
|
select {
|
||||||
|
case source := <-executed:
|
||||||
|
if source != fetcher.TriggerScheduled {
|
||||||
|
t.Fatal(source)
|
||||||
|
}
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("overdue job was not run")
|
||||||
|
}
|
||||||
|
entry := d.protocolJobs.jobs["test/refresh"]
|
||||||
|
entry.mu.Lock()
|
||||||
|
run := entry.active
|
||||||
|
entry.mu.Unlock()
|
||||||
|
if run != nil {
|
||||||
|
<-run.done
|
||||||
|
}
|
||||||
|
if err := d.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d = makeDownloader()
|
||||||
|
defer d.Close()
|
||||||
|
state := d.protocolJobs.jobs["test/refresh"].state
|
||||||
|
if state.LastSuccess.IsZero() {
|
||||||
|
t.Fatal("successful execution not restored")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-executed:
|
||||||
|
t.Fatal("recently successful job ran again on restart")
|
||||||
|
case <-time.After(30 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
+85
-16
@@ -2,6 +2,7 @@ package download
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -42,6 +43,7 @@ const (
|
|||||||
bucketTaskStats = "task_stats"
|
bucketTaskStats = "task_stats"
|
||||||
// protocol-level shared client state bucket
|
// protocol-level shared client state bucket
|
||||||
bucketProtocolState = "protocol_state"
|
bucketProtocolState = "protocol_state"
|
||||||
|
bucketProtocolJobs = "protocol_jobs"
|
||||||
// downloader config bucket
|
// downloader config bucket
|
||||||
bucketConfig = "config"
|
bucketConfig = "config"
|
||||||
// downloader extension bucket
|
// downloader extension bucket
|
||||||
@@ -99,6 +101,9 @@ type Progress struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Downloader struct {
|
type Downloader struct {
|
||||||
|
backgroundCtx context.Context
|
||||||
|
cancelBackground context.CancelFunc
|
||||||
|
protocolJobs *protocolScheduler
|
||||||
tempFiles *tempfiles.Scope
|
tempFiles *tempfiles.Scope
|
||||||
webviewProfilesLock sync.Mutex
|
webviewProfilesLock sync.Mutex
|
||||||
webviewProfiles map[string]*extensionWebViewProfile
|
webviewProfiles map[string]*extensionWebViewProfile
|
||||||
@@ -125,8 +130,10 @@ type Downloader struct {
|
|||||||
// Key: fullBaseName (e.g., "/path/archive.7z"), Value: taskID that claimed it
|
// Key: fullBaseName (e.g., "/path/archive.7z"), Value: taskID that claimed it
|
||||||
claimedExtractions sync.Map
|
claimedExtractions sync.Map
|
||||||
|
|
||||||
extensions []*Extension
|
extensions []*Extension
|
||||||
blob *internalblob.Registry
|
extensionGitLock sync.Mutex
|
||||||
|
extensionGitWG sync.WaitGroup
|
||||||
|
blob *internalblob.Registry
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewDownloader(cfg *DownloaderConfig) *Downloader {
|
func NewDownloader(cfg *DownloaderConfig) *Downloader {
|
||||||
@@ -135,11 +142,14 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
|
|||||||
}
|
}
|
||||||
cfg.Init()
|
cfg.Init()
|
||||||
|
|
||||||
|
backgroundCtx, cancelBackground := context.WithCancel(context.Background())
|
||||||
d := &Downloader{
|
d := &Downloader{
|
||||||
cfg: cfg,
|
backgroundCtx: backgroundCtx,
|
||||||
fetcherCache: make(map[string]fetcher.Fetcher),
|
cancelBackground: cancelBackground,
|
||||||
waitTasks: make([]*Task, 0),
|
cfg: cfg,
|
||||||
storage: cfg.Storage,
|
fetcherCache: make(map[string]fetcher.Fetcher),
|
||||||
|
waitTasks: make([]*Task, 0),
|
||||||
|
storage: cfg.Storage,
|
||||||
|
|
||||||
lock: &sync.Mutex{},
|
lock: &sync.Mutex{},
|
||||||
configLock: &sync.RWMutex{},
|
configLock: &sync.RWMutex{},
|
||||||
@@ -165,7 +175,7 @@ func (d *Downloader) Setup() error {
|
|||||||
d.blob = internalblob.NewRegistry("")
|
d.blob = internalblob.NewRegistry("")
|
||||||
|
|
||||||
// setup storage
|
// setup storage
|
||||||
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketTaskStats, bucketProtocolState, bucketConfig, bucketExtension, bucketExtensionStorage}); err != nil {
|
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketTaskStats, bucketProtocolState, bucketProtocolJobs, bucketConfig, bucketExtension, bucketExtensionStorage}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// load config from storage
|
// load config from storage
|
||||||
@@ -181,14 +191,13 @@ func (d *Downloader) Setup() error {
|
|||||||
FirstLoad: true,
|
FirstLoad: true,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// init default config
|
// Initialize the current backend configuration schema and defaults.
|
||||||
d.cfg.DownloaderStoreConfig.Init()
|
if err := d.initProtocolConfigs(); err != nil {
|
||||||
// init protocol config, if not exist, use default config
|
return err
|
||||||
|
}
|
||||||
|
// Attach optional protocol state stores after configuration initialization.
|
||||||
for _, fm := range d.cfg.FetchManagers {
|
for _, fm := range d.cfg.FetchManagers {
|
||||||
protocol := fm.Name()
|
protocol := fm.Name()
|
||||||
if _, ok := d.cfg.DownloaderStoreConfig.ProtocolConfig[protocol]; !ok {
|
|
||||||
d.cfg.DownloaderStoreConfig.ProtocolConfig[protocol] = fm.DefaultConfig()
|
|
||||||
}
|
|
||||||
if sfm, ok := fm.(fetcher.StatefulFetcherManager); ok {
|
if sfm, ok := fm.(fetcher.StatefulFetcherManager); ok {
|
||||||
sfm.SetStateStore(&protocolStateStore{
|
sfm.SetStateStore(&protocolStateStore{
|
||||||
storage: d.storage,
|
storage: d.storage,
|
||||||
@@ -197,6 +206,10 @@ func (d *Downloader) Setup() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := d.setupProtocolJobs(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// load tasks from storage
|
// load tasks from storage
|
||||||
var tasks []*Task
|
var tasks []*Task
|
||||||
if err = d.storage.List(bucketTask, &tasks); err != nil {
|
if err = d.storage.List(bucketTask, &tasks); err != nil {
|
||||||
@@ -365,6 +378,7 @@ func (d *Downloader) Setup() error {
|
|||||||
time.Sleep(time.Millisecond * time.Duration(d.cfg.RefreshInterval))
|
time.Sleep(time.Millisecond * time.Duration(d.cfg.RefreshInterval))
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
d.protocolJobs.start()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1181,7 +1195,14 @@ func (d *Downloader) doDelete(task *Task, force bool) (err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *Downloader) Close() error {
|
func (d *Downloader) Close() error {
|
||||||
d.closed.Store(true)
|
if d.closed.Swap(true) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
d.cancelBackground()
|
||||||
|
d.waitForExtensionGit()
|
||||||
|
if d.protocolJobs != nil {
|
||||||
|
d.protocolJobs.close()
|
||||||
|
}
|
||||||
d.webviewProxyLock.Lock()
|
d.webviewProxyLock.Lock()
|
||||||
if d.webviewProxy != nil {
|
if d.webviewProxy != nil {
|
||||||
_ = d.webviewProxy.Close()
|
_ = d.webviewProxy.Close()
|
||||||
@@ -1336,7 +1357,7 @@ func (d *Downloader) GetTasksByFilter(filter *TaskFilter) []*Task {
|
|||||||
func (d *Downloader) GetConfig() (*base.DownloaderStoreConfig, error) {
|
func (d *Downloader) GetConfig() (*base.DownloaderStoreConfig, error) {
|
||||||
d.configLock.RLock()
|
d.configLock.RLock()
|
||||||
defer d.configLock.RUnlock()
|
defer d.configLock.RUnlock()
|
||||||
return d.cfg.DownloaderStoreConfig, nil
|
return util.DeepClone(d.cfg.DownloaderStoreConfig), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error {
|
func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error {
|
||||||
@@ -1344,12 +1365,60 @@ func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error {
|
|||||||
return errors.New("config is nil")
|
return errors.New("config is nil")
|
||||||
}
|
}
|
||||||
next := util.DeepClone(v)
|
next := util.DeepClone(v)
|
||||||
|
changedProtocols := []string{}
|
||||||
d.configLock.Lock()
|
d.configLock.Lock()
|
||||||
defer d.configLock.Unlock()
|
if d.closed.Load() {
|
||||||
|
d.configLock.Unlock()
|
||||||
|
return errors.New("downloader is closed")
|
||||||
|
}
|
||||||
|
if next.GithubMirror == nil {
|
||||||
|
next.GithubMirror = util.DeepClone(d.cfg.GithubMirror)
|
||||||
|
}
|
||||||
|
if err := next.GithubMirror.Validate(); err != nil {
|
||||||
|
d.configLock.Unlock()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if next.ProtocolConfig == nil {
|
||||||
|
next.ProtocolConfig = map[string]any{}
|
||||||
|
}
|
||||||
|
for name := range next.ProtocolConfig {
|
||||||
|
if _, err := d.protocolManager(name); err != nil {
|
||||||
|
d.configLock.Unlock()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, fm := range d.cfg.FetchManagers {
|
||||||
|
name := fm.Name()
|
||||||
|
value, exists := next.ProtocolConfig[name]
|
||||||
|
if !exists {
|
||||||
|
value = d.cfg.ProtocolConfig[name]
|
||||||
|
}
|
||||||
|
value, err := preserveProtocolState(value, d.cfg.ProtocolConfig[name])
|
||||||
|
if err == nil {
|
||||||
|
value, err = normalizeProtocol(fm, value)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
d.configLock.Unlock()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
next.ProtocolConfig[name] = value
|
||||||
|
oldJSON, _ := json.Marshal(d.cfg.ProtocolConfig[name])
|
||||||
|
newJSON, _ := json.Marshal(value)
|
||||||
|
if !bytes.Equal(oldJSON, newJSON) {
|
||||||
|
changedProtocols = append(changedProtocols, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
if err := d.storage.Put(bucketConfig, "config", next); err != nil {
|
if err := d.storage.Put(bucketConfig, "config", next); err != nil {
|
||||||
|
d.configLock.Unlock()
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
d.cfg.DownloaderStoreConfig = next
|
d.cfg.DownloaderStoreConfig = next
|
||||||
|
d.configLock.Unlock()
|
||||||
|
if d.protocolJobs != nil {
|
||||||
|
for _, name := range changedProtocols {
|
||||||
|
d.protocolJobs.changed(name)
|
||||||
|
}
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1241,7 +1241,7 @@ func TestDownloader_Protocol_Config(t *testing.T) {
|
|||||||
"trackerSubscribeUrls": []string{
|
"trackerSubscribeUrls": []string{
|
||||||
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best.txt",
|
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best.txt",
|
||||||
},
|
},
|
||||||
"trackers": []string{
|
"customTrackers": []string{
|
||||||
"udp://tracker.coppersurfer.tk:6969/announce",
|
"udp://tracker.coppersurfer.tk:6969/announce",
|
||||||
"udp://tracker.leechers-paradise.org:6969/announce",
|
"udp://tracker.leechers-paradise.org:6969/announce",
|
||||||
},
|
},
|
||||||
@@ -1261,8 +1261,20 @@ func TestDownloader_Protocol_Config(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !test.JsonEqual(storeCfg, newStoreCfg) {
|
if newStoreCfg.DownloadDir != storeCfg.DownloadDir || newStoreCfg.Extra["theme"] != "dark" {
|
||||||
t.Errorf("GetConfig() got = %v, want %v", test.ToJson(storeCfg), test.ToJson(newStoreCfg))
|
t.Fatalf("downloader config was not saved: %+v", newStoreCfg)
|
||||||
|
}
|
||||||
|
var savedHTTP struct {
|
||||||
|
Connections int `json:"connections"`
|
||||||
|
}
|
||||||
|
if err := util.MapToStruct(newStoreCfg.ProtocolConfig["http"], &savedHTTP); err != nil || savedHTTP.Connections != 4 {
|
||||||
|
t.Fatalf("HTTP config: %+v %v", savedHTTP, err)
|
||||||
|
}
|
||||||
|
var savedBT struct {
|
||||||
|
CustomTrackers []string `json:"customTrackers"`
|
||||||
|
}
|
||||||
|
if err := util.MapToStruct(newStoreCfg.ProtocolConfig["bt"], &savedBT); err != nil || len(savedBT.CustomTrackers) != 2 {
|
||||||
|
t.Fatalf("BT derived trackers: %+v %v", savedBT, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2002,6 +2014,9 @@ func TestDownloader_ContinueOnStartupUsesContinueAllSemantics(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
cfg.Extra = map[string]any{"autoStartTasks": true}
|
cfg.Extra = map[string]any{"autoStartTasks": true}
|
||||||
|
if err := downloader.PutConfig(cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if err := downloader.ContinueOnStartup(); err != nil {
|
if err := downloader.ContinueOnStartup(); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -2010,6 +2025,9 @@ func TestDownloader_ContinueOnStartupUsesContinueAllSemantics(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cfg.AutoStartTasks = true
|
cfg.AutoStartTasks = true
|
||||||
|
if err := downloader.PutConfig(cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if err := downloader.ContinueOnStartup(); err != nil {
|
if err := downloader.ContinueOnStartup(); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
+73
-36
@@ -1,17 +1,19 @@
|
|||||||
package download
|
package download
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
|
"github.com/GopeedLab/gopeed/internal/githubmirror"
|
||||||
"github.com/GopeedLab/gopeed/internal/logger"
|
"github.com/GopeedLab/gopeed/internal/logger"
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
"github.com/GopeedLab/gopeed/pkg/download/engine"
|
"github.com/GopeedLab/gopeed/pkg/download/engine"
|
||||||
@@ -196,50 +198,85 @@ func (d *Downloader) getExtension(identity string) *Extension {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Downloader) fetchExtensionByGit(url string, handler func(tempExtPath string, devMode bool) (*Extension, error)) (*Extension, error) {
|
// waitForExtensionGit waits for clones and their install/check handlers before
|
||||||
if !strings.HasPrefix(url, "https://") && !strings.HasPrefix(url, "http://") {
|
// shutdown releases storage and temporary files. Close cancels backgroundCtx first.
|
||||||
url = "https://" + url
|
func (d *Downloader) waitForExtensionGit() {
|
||||||
}
|
d.extensionGitLock.Lock()
|
||||||
|
d.extensionGitLock.Unlock()
|
||||||
|
d.extensionGitWG.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// resolve project path
|
func (d *Downloader) fetchExtensionByGit(rawURL string, handler func(string, bool) (*Extension, error)) (*Extension, error) {
|
||||||
parentPath, projectPath := path.Split(url)
|
// Pair admission with shutdown so no operation is added after its wait starts.
|
||||||
// resolve project name and sub path
|
d.extensionGitLock.Lock()
|
||||||
pathArr := strings.SplitN(projectPath, "#", 2)
|
if d.closed.Load() {
|
||||||
projectPath = strings.TrimSuffix(pathArr[0], gitSuffix)
|
d.extensionGitLock.Unlock()
|
||||||
subPath := ""
|
return nil, fmt.Errorf("downloader is closed")
|
||||||
if len(pathArr) > 1 {
|
|
||||||
subPath = pathArr[1]
|
|
||||||
}
|
}
|
||||||
|
d.extensionGitWG.Add(1)
|
||||||
// Use unique suffix to avoid concurrent conflicts when multiple git operations
|
d.extensionGitLock.Unlock()
|
||||||
// target the same extension (e.g., install and update check happening simultaneously)
|
defer d.extensionGitWG.Done()
|
||||||
tempExtDir, err := os.MkdirTemp(d.cfg.TempDir, "gopeed-extension-")
|
ctx := d.backgroundCtx
|
||||||
|
if !strings.HasPrefix(rawURL, "https://") && !strings.HasPrefix(rawURL, "http://") {
|
||||||
|
rawURL = "https://" + rawURL
|
||||||
|
}
|
||||||
|
parsed, err := url.Parse(rawURL)
|
||||||
|
if err != nil || parsed.Host == "" {
|
||||||
|
return nil, fmt.Errorf("invalid extension repository URL")
|
||||||
|
}
|
||||||
|
subPath := filepath.FromSlash(parsed.Fragment)
|
||||||
|
if filepath.IsAbs(subPath) || subPath == ".." || strings.HasPrefix(filepath.Clean(subPath), ".."+string(filepath.Separator)) {
|
||||||
|
return nil, fmt.Errorf("invalid extension directory")
|
||||||
|
}
|
||||||
|
parsed.Fragment = ""
|
||||||
|
parsed.Path = strings.TrimSuffix(strings.TrimRight(parsed.Path, "/"), gitSuffix) + gitSuffix
|
||||||
|
officialURL := parsed.String()
|
||||||
|
cfg, err := d.GetConfig()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := d.tempFiles.Track(tempExtDir); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer d.tempFiles.Remove(tempExtDir)
|
|
||||||
|
|
||||||
proxyOptions := transport.ProxyOptions{}
|
proxyOptions := transport.ProxyOptions{}
|
||||||
proxyUrl := d.cfg.DownloaderStoreConfig.Proxy.ToUrl()
|
if proxyURL := cfg.Proxy.ToUrl(); proxyURL != nil {
|
||||||
if proxyUrl != nil {
|
proxyOptions.URL = proxyURL.Scheme + "://" + proxyURL.Host
|
||||||
proxyOptions.URL = proxyUrl.Scheme + "://" + proxyUrl.Host
|
if proxyURL.User != nil {
|
||||||
proxyOptions.Username = proxyUrl.User.Username()
|
proxyOptions.Username = proxyURL.User.Username()
|
||||||
proxyOptions.Password, _ = proxyUrl.User.Password()
|
proxyOptions.Password, _ = proxyURL.User.Password()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// clone project to extension temp dir
|
tempDir, err := os.MkdirTemp(d.cfg.TempDir, "gopeed-extension-")
|
||||||
gitUrl := parentPath + projectPath + gitSuffix
|
if err != nil {
|
||||||
if _, err := git.PlainClone(tempExtDir, false, &git.CloneOptions{
|
|
||||||
URL: gitUrl,
|
|
||||||
Depth: 1,
|
|
||||||
ProxyOptions: proxyOptions,
|
|
||||||
}); err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if err = d.tempFiles.Track(tempDir); err != nil {
|
||||||
return handler(filepath.Join(tempExtDir, subPath), false)
|
_ = os.RemoveAll(tempDir)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer d.tempFiles.Remove(tempDir)
|
||||||
|
repoDir := filepath.Join(tempDir, "repo")
|
||||||
|
repo, err := githubmirror.Fallback(ctx, cfg.GithubMirror, officialURL,
|
||||||
|
githubmirror.Options[*git.Repository]{Kind: githubmirror.Repository},
|
||||||
|
func(ctx context.Context, candidate string) (*git.Repository, error) {
|
||||||
|
if err := os.RemoveAll(repoDir); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return git.PlainCloneContext(ctx, repoDir, false, &git.CloneOptions{URL: candidate, Depth: 1, ProxyOptions: proxyOptions})
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Preserve the canonical source instead of persisting a selected mirror.
|
||||||
|
repoConfig, err := repo.Config()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if remote := repoConfig.Remotes["origin"]; remote != nil {
|
||||||
|
remote.URLs = []string{officialURL}
|
||||||
|
}
|
||||||
|
if err = repo.SetConfig(repoConfig); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return handler(filepath.Join(repoDir, subPath), false)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Downloader) parseExtensionByPath(path string) (*Extension, error) {
|
func (d *Downloader) parseExtensionByPath(path string) (*Extension, error) {
|
||||||
|
|||||||
@@ -0,0 +1,210 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/cgi"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Exercise real Git clone/install/check/update without depending on GitHub or
|
||||||
|
// public mirror availability. The proxy deliberately rejects the official URL.
|
||||||
|
// github.com/gopeed-test/repo is a placeholder: the successful mirror serves
|
||||||
|
// the temporary local repository through git http-backend instead.
|
||||||
|
func newExtensionGitFixture(t *testing.T) (*Downloader, func(string), *atomic.Int32, *atomic.Int32) {
|
||||||
|
t.Helper()
|
||||||
|
git, err := exec.LookPath("git")
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("Git executable required for HTTP Git fixture")
|
||||||
|
}
|
||||||
|
root := t.TempDir()
|
||||||
|
bare := filepath.Join(root, "repo.git")
|
||||||
|
work := t.TempDir()
|
||||||
|
command := func(args ...string) {
|
||||||
|
t.Helper()
|
||||||
|
cmd := exec.Command(git, args...)
|
||||||
|
if output, err := cmd.CombinedOutput(); err != nil {
|
||||||
|
t.Fatalf("git %v: %v: %s", args, err, output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
command("-c", "init.defaultBranch=main", "init", "--bare", bare)
|
||||||
|
command("clone", bare, work)
|
||||||
|
command("-C", work, "config", "user.name", "Gopeed Test")
|
||||||
|
command("-C", work, "config", "user.email", "[email protected]")
|
||||||
|
publish := func(version string) {
|
||||||
|
t.Helper()
|
||||||
|
directory := filepath.Join(work, "nested", "extension")
|
||||||
|
if err := os.MkdirAll(directory, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
manifest := map[string]any{"name": "mirror-test", "author": "test", "title": "Mirror test", "version": version, "repository": map[string]string{"url": "https://github.com/gopeed-test/repo", "directory": "nested/extension"}}
|
||||||
|
body, _ := json.Marshal(manifest)
|
||||||
|
if err := os.WriteFile(filepath.Join(directory, "manifest.json"), body, 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
command("-C", work, "add", ".")
|
||||||
|
command("-C", work, "commit", "-m", version)
|
||||||
|
command("-C", work, "push", "origin", "HEAD:main")
|
||||||
|
}
|
||||||
|
publish("1.0.0")
|
||||||
|
var official, badMirror atomic.Int32
|
||||||
|
backend := &cgi.Handler{Path: git, Args: []string{"http-backend"}, Env: []string{"GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}}
|
||||||
|
mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
r.URL.Path = strings.TrimPrefix(r.URL.Path, "/https://github.com/gopeed-test")
|
||||||
|
r.RequestURI = r.URL.RequestURI()
|
||||||
|
backend.ServeHTTP(w, r)
|
||||||
|
}))
|
||||||
|
t.Cleanup(mirror.Close)
|
||||||
|
failed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { badMirror.Add(1); http.Error(w, "mirror offline", 502) }))
|
||||||
|
t.Cleanup(failed.Close)
|
||||||
|
direct := &http.Transport{}
|
||||||
|
t.Cleanup(direct.CloseIdleConnections)
|
||||||
|
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method == http.MethodConnect || r.URL.Hostname() == "github.com" {
|
||||||
|
official.Add(1)
|
||||||
|
http.Error(w, "official unavailable", 502)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
request := r.Clone(r.Context())
|
||||||
|
request.RequestURI = ""
|
||||||
|
response, err := direct.RoundTrip(request)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), 502)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
for key, values := range response.Header {
|
||||||
|
for _, value := range values {
|
||||||
|
w.Header().Add(key, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.WriteHeader(response.StatusCode)
|
||||||
|
_, _ = io.Copy(w, response.Body)
|
||||||
|
}))
|
||||||
|
t.Cleanup(proxy.Close)
|
||||||
|
d := testBackendDownloader(t, &httpProtocol.FetcherManager{})
|
||||||
|
config, _ := d.GetConfig()
|
||||||
|
config.Proxy = &base.DownloaderProxyConfig{Enable: true, Scheme: "http", Host: strings.TrimPrefix(proxy.URL, "http://")}
|
||||||
|
config.GithubMirror = &base.GithubMirrorConfig{Enabled: true, Mirrors: []base.GithubMirror{{Type: "ghProxy", URL: failed.URL}, {Type: "ghProxy", URL: mirror.URL}}}
|
||||||
|
if err := d.PutConfig(config); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return d, publish, &official, &badMirror
|
||||||
|
}
|
||||||
|
func testInstallExtensionGitURL(t *testing.T, raw string) {
|
||||||
|
d, _, official, badMirror := newExtensionGitFixture(t)
|
||||||
|
extension, err := d.InstallExtensionByGit(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if extension.Version != "1.0.0" || extension.Identity != "test@mirror-test" {
|
||||||
|
t.Fatalf("%+v", extension)
|
||||||
|
}
|
||||||
|
if official.Load() == 0 || badMirror.Load() == 0 {
|
||||||
|
t.Fatal("ordered fallback was not exercised")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(d.ExtensionPath(extension), "manifest.json")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestExtensionGitMirrorInstallAndUpdate(t *testing.T) {
|
||||||
|
d, publish, official, badMirror := newExtensionGitFixture(t)
|
||||||
|
extension, err := d.InstallExtensionByGit("https://github.com/gopeed-test/repo#nested/extension")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
publish("1.1.0")
|
||||||
|
version, err := d.UpgradeCheckExtension(extension.Identity)
|
||||||
|
if err != nil || version != "1.1.0" {
|
||||||
|
t.Fatalf("%s %v", version, err)
|
||||||
|
}
|
||||||
|
if err = d.UpgradeExtension(extension.Identity); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
updated, err := d.GetExtension(extension.Identity)
|
||||||
|
if err != nil || updated.Version != "1.1.0" {
|
||||||
|
t.Fatalf("%+v %v", updated, err)
|
||||||
|
}
|
||||||
|
if updated.Repository.Url != "https://github.com/gopeed-test/repo" {
|
||||||
|
t.Fatal("mirror URL replaced canonical repository")
|
||||||
|
}
|
||||||
|
if official.Load() != 3 || badMirror.Load() != 3 {
|
||||||
|
t.Fatalf("attempts official=%d mirror=%d", official.Load(), badMirror.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtensionGitShutdownWaitsForHandler(t *testing.T) {
|
||||||
|
d, _, _, _ := newExtensionGitFixture(t)
|
||||||
|
started := make(chan string, 1)
|
||||||
|
release := make(chan struct{})
|
||||||
|
t.Cleanup(func() {
|
||||||
|
select {
|
||||||
|
case <-release:
|
||||||
|
default:
|
||||||
|
close(release)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
finished := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := d.fetchExtensionByGit("https://github.com/gopeed-test/repo#nested/extension", func(path string, _ bool) (*Extension, error) {
|
||||||
|
started <- path
|
||||||
|
<-release
|
||||||
|
return d.parseExtensionByPath(path)
|
||||||
|
})
|
||||||
|
finished <- err
|
||||||
|
}()
|
||||||
|
var path string
|
||||||
|
select {
|
||||||
|
case path = <-started:
|
||||||
|
case err := <-finished:
|
||||||
|
t.Fatalf("Git operation finished before handler: %v", err)
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("Git handler did not start")
|
||||||
|
}
|
||||||
|
closed := make(chan error, 1)
|
||||||
|
go func() { closed <- d.Close() }()
|
||||||
|
select {
|
||||||
|
case <-d.backgroundCtx.Done():
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("shutdown did not cancel Git context")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-closed:
|
||||||
|
t.Fatalf("shutdown returned before handler finished: %v", err)
|
||||||
|
case <-time.After(100 * time.Millisecond):
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(path, "manifest.json")); err != nil {
|
||||||
|
t.Fatalf("shutdown removed files still used by handler: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := d.InstallExtensionByGit("https://github.com/gopeed-test/repo"); err == nil || !strings.Contains(err.Error(), "downloader is closed") {
|
||||||
|
t.Fatalf("expected new Git operations to be rejected during shutdown: %v", err)
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
select {
|
||||||
|
case err := <-finished:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("Git handler did not finish")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-closed:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("shutdown did not finish after Git handler")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
gitclient "github.com/go-git/go-git/v5/plumbing/transport/client"
|
||||||
|
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
// go-git exposes a process-wide protocol registry. Register once during
|
||||||
|
// initialization, rather than changing it during concurrent clones. Proxy
|
||||||
|
// settings still come from each clone's ProxyOptions.
|
||||||
|
transport := githttp.NewClient(newExtensionGitHTTPClient(10*time.Second, 60*time.Second))
|
||||||
|
gitclient.InstallProtocol("http", transport)
|
||||||
|
gitclient.InstallProtocol("https", transport)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newExtensionGitHTTPClient(connectTimeout, idleTimeout time.Duration) *http.Client {
|
||||||
|
dialer := &net.Dialer{Timeout: connectTimeout, KeepAlive: 30 * time.Second}
|
||||||
|
// A socket read deadline must apply to one transfer, rather than a shared
|
||||||
|
// HTTP/2 connection carrying traffic from other requests.
|
||||||
|
protocols := new(http.Protocols)
|
||||||
|
protocols.SetHTTP1(true)
|
||||||
|
return &http.Client{Transport: &http.Transport{
|
||||||
|
Proxy: http.ProxyFromEnvironment,
|
||||||
|
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||||
|
conn, err := dialer.DialContext(ctx, network, address)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &gitIdleConn{Conn: conn, timeout: idleTimeout}, nil
|
||||||
|
},
|
||||||
|
TLSHandshakeTimeout: connectTimeout,
|
||||||
|
IdleConnTimeout: idleTimeout,
|
||||||
|
Protocols: protocols,
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
type gitIdleConn struct {
|
||||||
|
net.Conn
|
||||||
|
timeout time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *gitIdleConn) Read(p []byte) (int, error) {
|
||||||
|
if err := c.Conn.SetReadDeadline(time.Now().Add(c.timeout)); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return c.Conn.Read(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Refresh an already-pending read when a pooled HTTP/1 connection starts the
|
||||||
|
// next request; time spent idle in the pool must not consume its read allowance.
|
||||||
|
func (c *gitIdleConn) Write(p []byte) (int, error) {
|
||||||
|
if err := c.Conn.SetReadDeadline(time.Now().Add(c.timeout)); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return c.Conn.Write(p)
|
||||||
|
}
|
||||||
@@ -0,0 +1,189 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/http/httptrace"
|
||||||
|
"net/url"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/internal/githubmirror"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExtensionGitHTTPTransportAllowsSlowStreaming(t *testing.T) {
|
||||||
|
const chunks = 24
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
for range chunks {
|
||||||
|
if _, err := io.WriteString(w, "x"); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
select {
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
case <-r.Context().Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
client := newExtensionGitHTTPClient(time.Second, time.Second)
|
||||||
|
defer client.CloseIdleConnections()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
|
||||||
|
response, err := client.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
// A regularly streaming response lasts longer than the idle allowance.
|
||||||
|
body, err := io.ReadAll(response.Body)
|
||||||
|
if err != nil || len(body) != chunks {
|
||||||
|
t.Fatalf("received %d bytes, error=%v", len(body), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtensionGitHTTPTransportTimesOutStalledReads(t *testing.T) {
|
||||||
|
for _, bodyStarted := range []bool{false, true} {
|
||||||
|
name := "headers"
|
||||||
|
if bodyStarted {
|
||||||
|
name = "body"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if bodyStarted {
|
||||||
|
_, _ = io.WriteString(w, "partial")
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
}
|
||||||
|
<-r.Context().Done()
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
client := newExtensionGitHTTPClient(time.Second, 100*time.Millisecond)
|
||||||
|
defer client.CloseIdleConnections()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
|
||||||
|
response, err := client.Do(request)
|
||||||
|
if err == nil {
|
||||||
|
defer response.Body.Close()
|
||||||
|
_, err = io.ReadAll(response.Body)
|
||||||
|
}
|
||||||
|
var timeout net.Error
|
||||||
|
if !errors.As(err, &timeout) || !timeout.Timeout() || ctx.Err() != nil {
|
||||||
|
t.Fatalf("expected idle timeout before parent deadline: error=%v context=%v", err, ctx.Err())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtensionGitIdleTimeoutFallsBackThroughProxy(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Hostname() == "github.com" {
|
||||||
|
_, _ = io.WriteString(w, "partial")
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
<-r.Context().Done()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = io.WriteString(w, "complete")
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
client := newExtensionGitHTTPClient(time.Second, 100*time.Millisecond)
|
||||||
|
defer client.CloseIdleConnections()
|
||||||
|
proxy, _ := url.Parse(server.URL)
|
||||||
|
client.Transport.(*http.Transport).Proxy = http.ProxyURL(proxy)
|
||||||
|
config := &githubmirror.Config{Enabled: true, Mirrors: []githubmirror.Mirror{{Type: "ghProxy", URL: server.URL}}}
|
||||||
|
source := "http://github.com/test/repo.git"
|
||||||
|
var tried []string
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
result, err := githubmirror.Fallback(ctx, config, source, githubmirror.Options[string]{Kind: githubmirror.Repository}, func(ctx context.Context, candidate string) (string, error) {
|
||||||
|
tried = append(tried, candidate)
|
||||||
|
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, candidate, nil)
|
||||||
|
response, err := client.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
body, err := io.ReadAll(response.Body)
|
||||||
|
return string(body), err
|
||||||
|
})
|
||||||
|
if err != nil || result != "complete" || !reflect.DeepEqual(tried, []string{source, server.URL + "/" + source}) {
|
||||||
|
t.Fatalf("result=%q tried=%v error=%v", result, tried, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtensionGitHTTPTransportCancelsStalledBody(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
<-r.Context().Done()
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
client := newExtensionGitHTTPClient(time.Second, time.Minute)
|
||||||
|
defer client.CloseIdleConnections()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
|
||||||
|
response, err := client.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
cancel()
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { _, err := io.ReadAll(response.Body); done <- err }()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("error=%v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("parent cancellation did not interrupt the body")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtensionGitHTTPTransportRefreshesReusedConnection(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path == "/slow" {
|
||||||
|
select {
|
||||||
|
case <-time.After(500 * time.Millisecond):
|
||||||
|
case <-r.Context().Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, _ = io.WriteString(w, "ok")
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
client := newExtensionGitHTTPClient(time.Second, time.Second)
|
||||||
|
defer client.CloseIdleConnections()
|
||||||
|
response, err := client.Get(server.URL)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err = io.Copy(io.Discard, response.Body)
|
||||||
|
_ = response.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(700 * time.Millisecond)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
reused := false
|
||||||
|
ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{GotConn: func(info httptrace.GotConnInfo) { reused = info.Reused }})
|
||||||
|
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL+"/slow", nil)
|
||||||
|
response, err = client.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
body, err := io.ReadAll(response.Body)
|
||||||
|
if err != nil || string(body) != "ok" || !reused {
|
||||||
|
t.Fatalf("reused=%v body=%q error=%v", reused, body, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1070,54 +1070,15 @@ func TestDownloader_Extension_BlobRecoverOnError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestDownloader_InstallExtensionByGit(t *testing.T) {
|
func TestDownloader_InstallExtensionByGit(t *testing.T) {
|
||||||
setupDownloader(func(downloader *Downloader) {
|
testInstallExtensionGitURL(t, "https://github.com/gopeed-test/repo#nested/extension")
|
||||||
if _, err := downloader.InstallExtensionByGit("https://github.com/GopeedLab/gopeed-extension-samples#github-release-sample"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
|
||||||
URL: "https://github.com/GopeedLab/gopeed/releases",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(rr.Res.Files) == 1 {
|
|
||||||
t.Fatal("resolve error")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDownloader_InstallExtensionByGitSimple(t *testing.T) {
|
func TestDownloader_InstallExtensionByGitSimple(t *testing.T) {
|
||||||
setupDownloader(func(downloader *Downloader) {
|
testInstallExtensionGitURL(t, "github.com/gopeed-test/repo#nested/extension")
|
||||||
if _, err := downloader.InstallExtensionByGit("github.com/GopeedLab/gopeed-extension-samples#github-release-sample"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
|
||||||
URL: "https://github.com/GopeedLab/gopeed/releases",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(rr.Res.Files) == 1 {
|
|
||||||
t.Fatal("resolve error")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDownloader_InstallExtensionByGitFull(t *testing.T) {
|
func TestDownloader_InstallExtensionByGitFull(t *testing.T) {
|
||||||
setupDownloader(func(downloader *Downloader) {
|
testInstallExtensionGitURL(t, "https://github.com/gopeed-test/repo.git#nested/extension")
|
||||||
if _, err := downloader.InstallExtensionByGit("https://github.com/GopeedLab/gopeed-extension-samples.git#github-release-sample"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
|
||||||
URL: "https://github.com/GopeedLab/gopeed/releases",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(rr.Res.Files) == 1 {
|
|
||||||
t.Fatal("resolve error")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDownloader_UpgradeExtension(t *testing.T) {
|
func TestDownloader_UpgradeExtension(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,151 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/internal/githubmirror"
|
||||||
|
"github.com/GopeedLab/gopeed/internal/httpclient"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (d *Downloader) resourceClient(publicOnly bool) (*http.Client, error) {
|
||||||
|
cfg, err := d.GetConfig()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
options := httpclient.Options{
|
||||||
|
Client: httpclient.ClientOptions{Timeout: 30 * time.Second},
|
||||||
|
Transport: httpclient.TransportOptions{Proxy: cfg.Proxy.ToHandler(), TLSHandshakeTimeout: 10 * time.Second},
|
||||||
|
}
|
||||||
|
if publicOnly {
|
||||||
|
// A configured upstream proxy is trusted, but target URLs are still checked.
|
||||||
|
options.Client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||||
|
if len(via) >= 5 {
|
||||||
|
return errors.New("too many redirects")
|
||||||
|
}
|
||||||
|
return validatePublicURL(req.Context(), req.URL.String())
|
||||||
|
}
|
||||||
|
if cfg.Proxy == nil || !cfg.Proxy.Enable {
|
||||||
|
options.Transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||||
|
host, port, err := net.SplitHostPort(address)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
ips, err := publicIPs(ctx, host)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var last error
|
||||||
|
for _, ip := range ips {
|
||||||
|
conn, err := (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
|
||||||
|
if err == nil {
|
||||||
|
return conn, nil
|
||||||
|
}
|
||||||
|
last = err
|
||||||
|
}
|
||||||
|
return nil, last
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return httpclient.NewClient(options)
|
||||||
|
}
|
||||||
|
func publicIPs(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||||
|
ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(ips) == 0 {
|
||||||
|
return nil, errors.New("target has no addresses")
|
||||||
|
}
|
||||||
|
for _, ip := range ips {
|
||||||
|
if !ip.IP.IsGlobalUnicast() || ip.IP.IsPrivate() || ip.IP.IsLoopback() || ip.IP.IsLinkLocalUnicast() || ip.IP.IsUnspecified() {
|
||||||
|
return nil, errors.New("private or local resource target is not allowed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ips, nil
|
||||||
|
}
|
||||||
|
func validatePublicURL(ctx context.Context, raw string) error {
|
||||||
|
u, err := url.Parse(raw)
|
||||||
|
if err != nil || u == nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" || u.User != nil {
|
||||||
|
return errors.New("invalid resource URL")
|
||||||
|
}
|
||||||
|
_, err = publicIPs(ctx, u.Hostname())
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
func (d *Downloader) fetchPublicResource(ctx context.Context, raw string, limit int64, validate func([]byte) error) ([]byte, error) {
|
||||||
|
return d.fetchResource(ctx, raw, limit, true, validate)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchResource races complete, validated responses. A successful HEAD alone
|
||||||
|
// cannot win, and failed candidates never discard a usable official response.
|
||||||
|
func (d *Downloader) fetchResource(ctx context.Context, raw string, limit int64, publicOnly bool, validate func([]byte) error) ([]byte, error) {
|
||||||
|
cfg, err := d.GetConfig()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
client, err := d.resourceClient(publicOnly)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer client.CloseIdleConnections()
|
||||||
|
return githubmirror.Race(ctx, cfg.GithubMirror, raw,
|
||||||
|
githubmirror.Options[[]byte]{Kind: githubmirror.Resource, Timeout: 30 * time.Second},
|
||||||
|
func(ctx context.Context, candidate string) ([]byte, error) {
|
||||||
|
if publicOnly {
|
||||||
|
if err := validatePublicURL(ctx, candidate); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, candidate, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
response, err := client.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
if response.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("resource returned HTTP %d", response.StatusCode)
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(io.LimitReader(response.Body, limit+1))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if int64(len(body)) > limit {
|
||||||
|
return nil, errors.New("resource exceeds size limit")
|
||||||
|
}
|
||||||
|
if len(body) == 0 {
|
||||||
|
return nil, errors.New("empty resource")
|
||||||
|
}
|
||||||
|
if validate != nil {
|
||||||
|
if err := validate(body); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return body, nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// githubResourceCandidates accepts official GitHub HTTPS resource URLs only.
|
||||||
|
func githubResourceCandidates(config *githubmirror.Config, raw string) ([]string, error) {
|
||||||
|
u, err := url.Parse(raw)
|
||||||
|
if err != nil || u == nil || u.Scheme != "https" || u.User != nil || u.Port() != "" || u.Fragment != "" || strings.Contains(raw, "#") {
|
||||||
|
return nil, ErrInvalidMirrorURL
|
||||||
|
}
|
||||||
|
u.Host = strings.ToLower(u.Host)
|
||||||
|
switch u.Host {
|
||||||
|
case "github.com", "raw.githubusercontent.com":
|
||||||
|
default:
|
||||||
|
return nil, ErrInvalidMirrorURL
|
||||||
|
}
|
||||||
|
return githubmirror.Candidates(config, u.String(), githubmirror.Resource), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrMirrorProviderNotFound = errors.New("mirror provider not found")
|
||||||
|
ErrInvalidMirrorURL = errors.New("invalid mirror resource URL")
|
||||||
|
)
|
||||||
|
|
||||||
|
// Each provider owns URL validation and candidate generation using its own
|
||||||
|
// configuration. Providers only generate URLs; they must not fetch resources.
|
||||||
|
type mirrorCandidateProvider func(config *base.DownloaderStoreConfig, raw string) ([]string, error)
|
||||||
|
|
||||||
|
// Add providers here without changing the public API or its client transport.
|
||||||
|
var mirrorCandidateProviders = map[string]mirrorCandidateProvider{
|
||||||
|
"github": func(config *base.DownloaderStoreConfig, raw string) ([]string, error) {
|
||||||
|
return githubResourceCandidates(config.GithubMirror, raw)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Downloader) GetMirrorCandidates(provider, raw string) ([]string, error) {
|
||||||
|
candidates, ok := mirrorCandidateProviders[provider]
|
||||||
|
if !ok {
|
||||||
|
return nil, ErrMirrorProviderNotFound
|
||||||
|
}
|
||||||
|
config, err := d.GetConfig()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return candidates(config, raw)
|
||||||
|
}
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/util"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (d *Downloader) protocolManager(name string) (fetcher.FetcherManager, error) {
|
||||||
|
for _, fm := range d.cfg.FetchManagers {
|
||||||
|
if fm.Name() == name {
|
||||||
|
return fm, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("%w: %s", ErrUnSupportedProtocol, name)
|
||||||
|
}
|
||||||
|
func jsonObject(v any) (map[string]json.RawMessage, error) {
|
||||||
|
raw, err := json.Marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var object map[string]json.RawMessage
|
||||||
|
if err = json.Unmarshal(raw, &object); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if object == nil {
|
||||||
|
object = map[string]json.RawMessage{}
|
||||||
|
}
|
||||||
|
return object, nil
|
||||||
|
}
|
||||||
|
func normalizeProtocol(fm fetcher.FetcherManager, v any) (any, error) {
|
||||||
|
raw, err := json.Marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
target := fm.DefaultConfig()
|
||||||
|
if target == nil {
|
||||||
|
return nil, errors.New("protocol does not expose configuration")
|
||||||
|
}
|
||||||
|
// Managers may return either typed pointers or value/map defaults.
|
||||||
|
valueDefault := reflect.TypeOf(target).Kind() != reflect.Pointer
|
||||||
|
if valueDefault {
|
||||||
|
pointer := reflect.New(reflect.TypeOf(target))
|
||||||
|
defaults, err := json.Marshal(target)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err = json.Unmarshal(defaults, pointer.Interface()); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
target = pointer.Interface()
|
||||||
|
}
|
||||||
|
if err = json.Unmarshal(raw, target); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if normalizer, ok := fm.(fetcher.ConfigNormalizer); ok {
|
||||||
|
raw, err = json.Marshal(target)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return normalizer.NormalizeConfig(raw)
|
||||||
|
}
|
||||||
|
if valueDefault {
|
||||||
|
return reflect.ValueOf(target).Elem().Interface(), nil
|
||||||
|
}
|
||||||
|
return target, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// preserveProtocolState ignores all client-supplied state, including null or
|
||||||
|
// malformed state values, before decoding/validating the writable config.
|
||||||
|
func preserveProtocolState(next, previous any) (any, error) {
|
||||||
|
object, err := jsonObject(next)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
before, err := jsonObject(previous)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Match encoding/json field folding as well as the canonical spelling.
|
||||||
|
for key := range object {
|
||||||
|
if strings.EqualFold(key, "state") {
|
||||||
|
delete(object, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if state, ok := before["state"]; ok {
|
||||||
|
object["state"] = state
|
||||||
|
}
|
||||||
|
return object, nil
|
||||||
|
}
|
||||||
|
func (d *Downloader) readProtocolConfig(name string) (any, error) {
|
||||||
|
if _, err := d.protocolManager(name); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
d.configLock.RLock()
|
||||||
|
defer d.configLock.RUnlock()
|
||||||
|
value := d.cfg.ProtocolConfig[name]
|
||||||
|
return *util.DeepClone(&value), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// updateProtocolState is the only internal write path for backend-owned state.
|
||||||
|
// It never publishes callback edits outside the reserved state section.
|
||||||
|
func (d *Downloader) updateProtocolState(name string, change func(config, state json.RawMessage) (any, error)) error {
|
||||||
|
fm, err := d.protocolManager(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
d.configLock.Lock()
|
||||||
|
defer d.configLock.Unlock()
|
||||||
|
if d.closed.Load() {
|
||||||
|
return errors.New("downloader is closed")
|
||||||
|
}
|
||||||
|
next := util.DeepClone(d.cfg.DownloaderStoreConfig)
|
||||||
|
object, err := jsonObject(next.ProtocolConfig[name])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
state, ok := object["state"]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("protocol %s does not expose persistent state", name)
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(object)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
value, err := change(raw, state)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Keep the transaction's original config isolated from callback mutations.
|
||||||
|
object["state"], err = json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
normalized, err := normalizeProtocol(fm, object)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
next.ProtocolConfig[name] = normalized
|
||||||
|
if err = d.storage.Put(bucketConfig, "config", next); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
d.cfg.DownloaderStoreConfig.ProtocolConfig = next.ProtocolConfig
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// initProtocolConfigs loads the current schema and initializes missing defaults.
|
||||||
|
// This release intentionally does not import settings from older config layouts.
|
||||||
|
func (d *Downloader) initProtocolConfigs() error {
|
||||||
|
cfg := d.cfg.DownloaderStoreConfig
|
||||||
|
cfg.Init()
|
||||||
|
if err := cfg.GithubMirror.Validate(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, fm := range d.cfg.FetchManagers {
|
||||||
|
value, ok := cfg.ProtocolConfig[fm.Name()]
|
||||||
|
if !ok {
|
||||||
|
value = map[string]any{}
|
||||||
|
}
|
||||||
|
normalized, err := normalizeProtocol(fm, value)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%s config: %w", fm.Name(), err)
|
||||||
|
}
|
||||||
|
cfg.ProtocolConfig[fm.Name()] = normalized
|
||||||
|
}
|
||||||
|
return d.storage.Put(bucketConfig, "config", cfg)
|
||||||
|
}
|
||||||
@@ -0,0 +1,250 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/util"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrProtocolJobNotFound = errors.New("protocol job not found")
|
||||||
|
var jobNamePattern = regexp.MustCompile(`^[a-z][a-z0-9-]*$`)
|
||||||
|
|
||||||
|
type protocolJobState struct {
|
||||||
|
LastSuccess time.Time `json:"lastSuccess"`
|
||||||
|
Failures int `json:"failures"`
|
||||||
|
}
|
||||||
|
type protocolJobRun struct {
|
||||||
|
done chan struct{}
|
||||||
|
err error
|
||||||
|
source fetcher.TriggerSource
|
||||||
|
}
|
||||||
|
type registeredJob struct {
|
||||||
|
protocol string
|
||||||
|
spec fetcher.CronJob
|
||||||
|
mu sync.Mutex
|
||||||
|
state protocolJobState
|
||||||
|
active *protocolJobRun
|
||||||
|
next time.Time
|
||||||
|
wake chan struct{}
|
||||||
|
force atomic.Bool
|
||||||
|
}
|
||||||
|
type protocolScheduler struct {
|
||||||
|
d *Downloader
|
||||||
|
jobs map[string]*registeredJob
|
||||||
|
wg sync.WaitGroup
|
||||||
|
// gate prevents adding work while shutdown waits for it.
|
||||||
|
gate sync.Mutex
|
||||||
|
stopped bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Downloader) setupProtocolJobs() error {
|
||||||
|
scheduler := &protocolScheduler{d: d, jobs: map[string]*registeredJob{}}
|
||||||
|
for _, fm := range d.cfg.FetchManagers {
|
||||||
|
provider, ok := fm.(fetcher.CronProvider)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := fm.Name()
|
||||||
|
host := fetcher.ProtocolHost{
|
||||||
|
Read: func(dst any) error {
|
||||||
|
value, err := d.readProtocolConfig(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return util.MapToStruct(value, dst)
|
||||||
|
},
|
||||||
|
UpdateState: func(change func(config, state json.RawMessage) (any, error)) error {
|
||||||
|
return d.updateProtocolState(name, change)
|
||||||
|
},
|
||||||
|
Fetch: func(ctx context.Context, raw string, limit int64, validate func([]byte) error) ([]byte, error) {
|
||||||
|
return d.fetchPublicResource(ctx, raw, limit, validate)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, spec := range provider.CronJobs(host) {
|
||||||
|
if !jobNamePattern.MatchString(spec.Name) || spec.Every <= 0 || spec.Timeout <= 0 || spec.Run == nil {
|
||||||
|
return fmt.Errorf("invalid protocol job: %s/%s", name, spec.Name)
|
||||||
|
}
|
||||||
|
key := name + "/" + spec.Name
|
||||||
|
if _, ok := scheduler.jobs[key]; ok {
|
||||||
|
return fmt.Errorf("duplicate protocol job: %s", key)
|
||||||
|
}
|
||||||
|
entry := ®isteredJob{protocol: name, spec: spec, wake: make(chan struct{}, 1)}
|
||||||
|
if _, err := d.storage.Get(bucketProtocolJobs, key, &entry.state); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
scheduler.jobs[key] = entry
|
||||||
|
}
|
||||||
|
}
|
||||||
|
d.protocolJobs = scheduler
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (s *protocolScheduler) start() {
|
||||||
|
for key, entry := range s.jobs {
|
||||||
|
s.wg.Add(1)
|
||||||
|
go s.loop(key, entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (s *protocolScheduler) signal(entry *registeredJob) {
|
||||||
|
select {
|
||||||
|
case entry.wake <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (s *protocolScheduler) changed(protocol string) {
|
||||||
|
for _, entry := range s.jobs {
|
||||||
|
if protocol == "" || entry.protocol == protocol {
|
||||||
|
entry.force.Store(true)
|
||||||
|
s.signal(entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (s *protocolScheduler) loop(key string, entry *registeredJob) {
|
||||||
|
defer s.wg.Done()
|
||||||
|
entry.mu.Lock()
|
||||||
|
if entry.next.IsZero() {
|
||||||
|
entry.next = entry.state.LastSuccess.Add(entry.spec.Every)
|
||||||
|
if entry.next.Before(time.Now()) {
|
||||||
|
entry.next = time.Now().Add(time.Second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
next := entry.next
|
||||||
|
entry.mu.Unlock()
|
||||||
|
timer := time.NewTimer(time.Until(next))
|
||||||
|
defer timer.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-s.d.backgroundCtx.Done():
|
||||||
|
return
|
||||||
|
case <-entry.wake:
|
||||||
|
entry.mu.Lock()
|
||||||
|
next = entry.next
|
||||||
|
entry.mu.Unlock()
|
||||||
|
if entry.force.Swap(false) {
|
||||||
|
next = time.Now()
|
||||||
|
}
|
||||||
|
case <-timer.C:
|
||||||
|
_ = s.trigger(s.d.backgroundCtx, key, fetcher.TriggerScheduled)
|
||||||
|
entry.mu.Lock()
|
||||||
|
next = entry.next
|
||||||
|
entry.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
if !timer.Stop() {
|
||||||
|
select {
|
||||||
|
case <-timer.C:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
timer.Reset(max(time.Until(next), time.Millisecond))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (s *protocolScheduler) trigger(ctx context.Context, key string, source fetcher.TriggerSource) error {
|
||||||
|
entry, ok := s.jobs[key]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("%w: %s", ErrProtocolJobNotFound, key)
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.gate.Lock()
|
||||||
|
if s.stopped || s.d.closed.Load() {
|
||||||
|
s.gate.Unlock()
|
||||||
|
return errors.New("downloader is closed")
|
||||||
|
}
|
||||||
|
entry.mu.Lock()
|
||||||
|
run := entry.active
|
||||||
|
if run == nil {
|
||||||
|
run = &protocolJobRun{done: make(chan struct{}), source: source}
|
||||||
|
entry.active = run
|
||||||
|
s.wg.Add(1)
|
||||||
|
go s.execute(key, entry, run, source)
|
||||||
|
}
|
||||||
|
entry.mu.Unlock()
|
||||||
|
s.gate.Unlock()
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-run.done:
|
||||||
|
// A manual request must still run when the scheduled attempt it
|
||||||
|
// joined was skipped (for example, automatic refresh is disabled).
|
||||||
|
if source == fetcher.TriggerManual && run.source == fetcher.TriggerScheduled && errors.Is(run.err, fetcher.ErrJobSkipped) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return run.err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (s *protocolScheduler) execute(key string, entry *registeredJob, run *protocolJobRun, source fetcher.TriggerSource) {
|
||||||
|
defer s.wg.Done()
|
||||||
|
ctx, cancel := context.WithTimeout(s.d.backgroundCtx, entry.spec.Timeout)
|
||||||
|
defer cancel()
|
||||||
|
err := func() (err error) {
|
||||||
|
defer func() {
|
||||||
|
if v := recover(); v != nil {
|
||||||
|
err = fmt.Errorf("protocol job panic: %v", v)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return entry.spec.Run(ctx, source)
|
||||||
|
}()
|
||||||
|
if err == nil {
|
||||||
|
err = ctx.Err()
|
||||||
|
}
|
||||||
|
entry.mu.Lock()
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
entry.state.LastSuccess = time.Now().UTC()
|
||||||
|
entry.state.Failures = 0
|
||||||
|
case errors.Is(err, fetcher.ErrJobSkipped), errors.Is(err, fetcher.ErrConfigChanged), errors.Is(err, context.Canceled):
|
||||||
|
default:
|
||||||
|
entry.state.Failures++
|
||||||
|
}
|
||||||
|
if storeErr := s.d.storage.Put(bucketProtocolJobs, key, entry.state); storeErr != nil {
|
||||||
|
err = errors.Join(err, storeErr)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
entry.next = entry.state.LastSuccess.Add(entry.spec.Every)
|
||||||
|
case errors.Is(err, fetcher.ErrJobSkipped):
|
||||||
|
entry.next = time.Now().Add(entry.spec.Every)
|
||||||
|
case errors.Is(err, fetcher.ErrConfigChanged):
|
||||||
|
entry.next = time.Now().Add(time.Second)
|
||||||
|
default:
|
||||||
|
retry := time.Minute * time.Duration(1<<min(max(entry.state.Failures-1, 0), 6))
|
||||||
|
entry.next = time.Now().Add(retry)
|
||||||
|
}
|
||||||
|
run.err = err
|
||||||
|
entry.active = nil
|
||||||
|
close(run.done)
|
||||||
|
entry.mu.Unlock()
|
||||||
|
if err != nil && !errors.Is(err, fetcher.ErrJobSkipped) && !errors.Is(err, context.Canceled) {
|
||||||
|
s.d.Logger.Warn().Err(err).Str("job", key).Str("source", string(source)).Msg("protocol job failed")
|
||||||
|
}
|
||||||
|
if source == fetcher.TriggerManual {
|
||||||
|
s.signal(entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (s *protocolScheduler) close() {
|
||||||
|
s.gate.Lock()
|
||||||
|
s.stopped = true
|
||||||
|
s.gate.Unlock()
|
||||||
|
s.wg.Wait()
|
||||||
|
}
|
||||||
|
func (d *Downloader) TriggerProtocolJob(ctx context.Context, protocol, name string) error {
|
||||||
|
if d.protocolJobs == nil {
|
||||||
|
return ErrProtocolJobNotFound
|
||||||
|
}
|
||||||
|
err := d.protocolJobs.trigger(ctx, protocol+"/"+name, fetcher.TriggerManual)
|
||||||
|
if errors.Is(err, fetcher.ErrJobSkipped) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
|
)
|
||||||
|
|
||||||
|
type jobWaitContext struct {
|
||||||
|
context.Context
|
||||||
|
waiting chan struct{}
|
||||||
|
once sync.Once
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *jobWaitContext) Done() <-chan struct{} {
|
||||||
|
c.once.Do(func() { close(c.waiting) })
|
||||||
|
return c.Context.Done()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManualProtocolJobRunsAfterJoinedScheduledSkip(t *testing.T) {
|
||||||
|
entered := make(chan struct{})
|
||||||
|
release := make(chan struct{})
|
||||||
|
var manualCalls atomic.Int32
|
||||||
|
manager := &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Second,
|
||||||
|
Run: func(ctx context.Context, source fetcher.TriggerSource) error {
|
||||||
|
if source == fetcher.TriggerManual {
|
||||||
|
manualCalls.Add(1)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
close(entered)
|
||||||
|
select {
|
||||||
|
case <-release:
|
||||||
|
return fetcher.ErrJobSkipped
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
d := testBackendDownloader(t, manager)
|
||||||
|
scheduled := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
scheduled <- d.protocolJobs.trigger(context.Background(), "test/refresh", fetcher.TriggerScheduled)
|
||||||
|
}()
|
||||||
|
<-entered
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
waiter := &jobWaitContext{Context: ctx, waiting: make(chan struct{})}
|
||||||
|
manual := make(chan error, 1)
|
||||||
|
go func() { manual <- d.TriggerProtocolJob(waiter, "test", "refresh") }()
|
||||||
|
<-waiter.waiting
|
||||||
|
close(release)
|
||||||
|
if err := <-scheduled; !errors.Is(err, fetcher.ErrJobSkipped) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := <-manual; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if manualCalls.Load() != 1 {
|
||||||
|
t.Fatal("manual refresh was swallowed by skipped automatic work")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManualProtocolJobFailureKeepsScheduledBackoff(t *testing.T) {
|
||||||
|
var calls atomic.Int32
|
||||||
|
failure := errors.New("offline")
|
||||||
|
manager := &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Second,
|
||||||
|
Run: func(context.Context, fetcher.TriggerSource) error { calls.Add(1); return failure },
|
||||||
|
}}
|
||||||
|
d := testBackendDownloader(t, manager)
|
||||||
|
if err := d.TriggerProtocolJob(context.Background(), "test", "refresh"); !errors.Is(err, failure) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entry := d.protocolJobs.jobs["test/refresh"]
|
||||||
|
entry.mu.Lock()
|
||||||
|
next := entry.next
|
||||||
|
entry.mu.Unlock()
|
||||||
|
if time.Until(next) < 59*time.Second {
|
||||||
|
t.Fatal("manual failure did not schedule retry backoff")
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
if calls.Load() != 1 {
|
||||||
|
t.Fatalf("manual failure immediately retried %d times", calls.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,340 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
|
btProtocol "github.com/GopeedLab/gopeed/internal/protocol/bt"
|
||||||
|
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
|
)
|
||||||
|
|
||||||
|
// State tests initialize config without starting network clients or cron loops.
|
||||||
|
func protocolConfigDownloader(t *testing.T, storage Storage, managers ...fetcher.FetcherManager) *Downloader {
|
||||||
|
t.Helper()
|
||||||
|
if storage == nil {
|
||||||
|
storage = NewMemStorage()
|
||||||
|
}
|
||||||
|
d := NewDownloader(&DownloaderConfig{Storage: storage, StorageDir: t.TempDir(), FetchManagers: managers})
|
||||||
|
d.cfg.DownloaderStoreConfig = &base.DownloaderStoreConfig{}
|
||||||
|
if err := storage.Setup([]string{bucketConfig}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { d.cancelBackground(); _ = storage.Close() })
|
||||||
|
if err := d.initProtocolConfigs(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
func protocolObject(t *testing.T, d *Downloader, name string) map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
cfg, err := d.GetConfig()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(cfg.ProtocolConfig[name])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var result map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &result); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
func configureBT(t *testing.T, d *Downloader, urls ...string) {
|
||||||
|
t.Helper()
|
||||||
|
cfg, _ := d.GetConfig()
|
||||||
|
cfg.ProtocolConfig["bt"] = map[string]any{"autoUpdateTrackers": false, "trackerSubscribeUrls": urls, "customTrackers": []string{"udp://custom.example:80/announce"}}
|
||||||
|
if err := d.PutConfig(cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func seedBTState(t *testing.T, d *Downloader) {
|
||||||
|
t.Helper()
|
||||||
|
if err := d.updateProtocolState("bt", func(_, _ json.RawMessage) (any, error) {
|
||||||
|
return map[string]any{
|
||||||
|
"trackerSources": map[string][]string{"https://example.com/list": {"udp://cached.example:80/announce"}},
|
||||||
|
"trackerSubscriptionStatuses": map[string]any{"https://example.com/list": map[string]any{"success": true, "trackerCount": 1, "lastFetchTime": time.Date(2026, 10, 19, 1, 2, 3, 0, time.UTC)}},
|
||||||
|
"lastTrackerUpdateTime": time.Date(2026, 10, 19, 1, 2, 3, 0, time.UTC),
|
||||||
|
}, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFullConfigValidatesProtocolsAndPreservesExplicitValues(t *testing.T) {
|
||||||
|
d := protocolConfigDownloader(t, nil, &httpProtocol.FetcherManager{}, &btProtocol.FetcherManager{})
|
||||||
|
configureBT(t, d)
|
||||||
|
cfg, _ := d.GetConfig()
|
||||||
|
cfg.ProtocolConfig["http"] = map[string]any{"connections": 0, "useServerCtime": false}
|
||||||
|
if err := d.PutConfig(cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := protocolObject(t, d, "http")
|
||||||
|
if got["connections"] != float64(0) || got["useServerCtime"] != false {
|
||||||
|
t.Fatal(got)
|
||||||
|
}
|
||||||
|
for _, invalid := range []any{map[string]any{"connections": "bad"}, map[string]any{"useServerCtime": "bad"}} {
|
||||||
|
cfg, _ := d.GetConfig()
|
||||||
|
cfg.ProtocolConfig["http"] = invalid
|
||||||
|
if err := d.PutConfig(cfg); err == nil {
|
||||||
|
t.Fatalf("invalid protocol config accepted: %v", invalid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cfg, _ = d.GetConfig()
|
||||||
|
cfg.DownloadDir = "must-not-publish"
|
||||||
|
cfg.ProtocolConfig["bt"] = map[string]any{"listenPort": -1}
|
||||||
|
if err := d.PutConfig(cfg); err == nil {
|
||||||
|
t.Fatal("invalid BT config accepted")
|
||||||
|
}
|
||||||
|
saved, _ := d.GetConfig()
|
||||||
|
if saved.DownloadDir == "must-not-publish" {
|
||||||
|
t.Fatal("invalid full PUT partially published")
|
||||||
|
}
|
||||||
|
cfg, _ = d.GetConfig()
|
||||||
|
cfg.ProtocolConfig["unknown"] = map[string]any{"state": map[string]any{"injected": true}}
|
||||||
|
if err := d.PutConfig(cfg); err == nil {
|
||||||
|
t.Fatal("unregistered protocol accepted")
|
||||||
|
}
|
||||||
|
// Returned configs are detached snapshots.
|
||||||
|
snapshot, _ := d.GetConfig()
|
||||||
|
snapshot.ProtocolConfig["http"] = map[string]any{"connections": 999}
|
||||||
|
if !reflect.DeepEqual(protocolObject(t, d, "http"), got) {
|
||||||
|
t.Fatal("snapshot changed live config")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProtocolConfigIgnoresUnknownFields(t *testing.T) {
|
||||||
|
for _, mode := range []string{"stored", "put"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
d := protocolConfigDownloader(t, nil, &btProtocol.FetcherManager{})
|
||||||
|
cfg, _ := d.GetConfig()
|
||||||
|
cfg.ProtocolConfig["bt"] = map[string]any{
|
||||||
|
"listenPort": 6881,
|
||||||
|
"autoUpdateTrackers": false,
|
||||||
|
"trackers": []string{"udp://obsolete.example:80/announce"},
|
||||||
|
"state": map[string]any{"obsolete": true},
|
||||||
|
}
|
||||||
|
if mode == "stored" {
|
||||||
|
if err := d.storage.Put(bucketConfig, "config", cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var restored base.DownloaderStoreConfig
|
||||||
|
if exists, err := d.storage.Get(bucketConfig, "config", &restored); err != nil || !exists {
|
||||||
|
t.Fatalf("load stored config: %v %v", exists, err)
|
||||||
|
}
|
||||||
|
d.cfg.DownloaderStoreConfig = &restored
|
||||||
|
if err := d.initProtocolConfigs(); err != nil {
|
||||||
|
t.Fatalf("load config containing removed fields: %v", err)
|
||||||
|
}
|
||||||
|
} else if err := d.PutConfig(cfg); err != nil {
|
||||||
|
t.Fatalf("save config containing unknown fields: %v", err)
|
||||||
|
}
|
||||||
|
actual := protocolObject(t, d, "bt")
|
||||||
|
if actual["listenPort"] != float64(6881) || actual["autoUpdateTrackers"] != false {
|
||||||
|
t.Fatalf("known fields changed: %v", actual)
|
||||||
|
}
|
||||||
|
if _, exists := actual["trackers"]; exists {
|
||||||
|
t.Fatal("removed field retained")
|
||||||
|
}
|
||||||
|
if len(actual["customTrackers"].([]any)) != 0 {
|
||||||
|
t.Fatal("removed trackers migrated into custom trackers")
|
||||||
|
}
|
||||||
|
if _, exists := actual["state"].(map[string]any)["obsolete"]; exists {
|
||||||
|
t.Fatal("unknown state field retained")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBTStateSurvivesOmittedNullForgedAndStaleFullConfig(t *testing.T) {
|
||||||
|
for _, mode := range []string{"omitted", "null", "forged", "wrong-type", "stale", "omitted-protocol", "omitted-timestamp", "case-variant", "unicode-variant"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
d := protocolConfigDownloader(t, nil, &btProtocol.FetcherManager{})
|
||||||
|
configureBT(t, d, "https://example.com/list")
|
||||||
|
stale, _ := d.GetConfig()
|
||||||
|
seedBTState(t, d)
|
||||||
|
expected := protocolObject(t, d, "bt")["state"]
|
||||||
|
cfg, _ := d.GetConfig()
|
||||||
|
bt := protocolObject(t, d, "bt")
|
||||||
|
switch mode {
|
||||||
|
case "omitted":
|
||||||
|
delete(bt, "state")
|
||||||
|
case "null":
|
||||||
|
bt["state"] = nil
|
||||||
|
case "forged":
|
||||||
|
bt["state"] = map[string]any{"lastTrackerUpdateTime": "2000-01-01T00:00:00Z", "trackerSources": map[string]any{}, "unknown": true}
|
||||||
|
case "wrong-type":
|
||||||
|
bt["state"] = "ignore this before decoding"
|
||||||
|
case "stale":
|
||||||
|
cfg = stale
|
||||||
|
bt = map[string]any{"autoUpdateTrackers": false, "trackerSubscribeUrls": []string{"https://example.com/list"}, "customTrackers": []string{"udp://custom.example:80/announce"}, "state": map[string]any{}}
|
||||||
|
case "omitted-protocol":
|
||||||
|
delete(cfg.ProtocolConfig, "bt")
|
||||||
|
case "case-variant":
|
||||||
|
bt["STATE"] = "ignore this alias"
|
||||||
|
case "unicode-variant":
|
||||||
|
bt["ſtate"] = "ignore Unicode case folding"
|
||||||
|
case "omitted-timestamp":
|
||||||
|
bt["state"] = map[string]any{"trackerSources": map[string]any{}}
|
||||||
|
}
|
||||||
|
if mode != "omitted-protocol" {
|
||||||
|
bt["listenPort"] = 6881
|
||||||
|
cfg.ProtocolConfig["bt"] = bt
|
||||||
|
}
|
||||||
|
if err := d.PutConfig(cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
actual := protocolObject(t, d, "bt")
|
||||||
|
if !reflect.DeepEqual(actual["state"], expected) {
|
||||||
|
t.Fatalf("backend state replaced: %v", actual)
|
||||||
|
}
|
||||||
|
if _, exists := actual["trackers"]; exists {
|
||||||
|
t.Fatal("derived tracker list persisted")
|
||||||
|
}
|
||||||
|
if mode != "omitted-protocol" && actual["listenPort"] != float64(6881) {
|
||||||
|
t.Fatal("writable config did not change")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBTConfigRemovalPrunesOnlyRemovedState(t *testing.T) {
|
||||||
|
d := protocolConfigDownloader(t, nil, &btProtocol.FetcherManager{})
|
||||||
|
configureBT(t, d, "https://example.com/list")
|
||||||
|
seedBTState(t, d)
|
||||||
|
configureBT(t, d, "https://new.example/list")
|
||||||
|
state := protocolObject(t, d, "bt")["state"].(map[string]any)
|
||||||
|
if len(state["trackerSources"].(map[string]any)) != 0 || len(state["trackerSubscriptionStatuses"].(map[string]any)) != 0 {
|
||||||
|
t.Fatal("removed subscription retained state")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type counterState struct {
|
||||||
|
Count int `json:"count"`
|
||||||
|
Other int `json:"other"`
|
||||||
|
}
|
||||||
|
type counterConfig struct {
|
||||||
|
fetcher.ConfigState[counterState]
|
||||||
|
Writable int `json:"writable"`
|
||||||
|
}
|
||||||
|
type counterManager struct{ httpProtocol.FetcherManager }
|
||||||
|
|
||||||
|
func (*counterManager) Name() string { return "counter" }
|
||||||
|
func (*counterManager) DefaultConfig() any { return &counterConfig{Writable: 7} }
|
||||||
|
func counterHost(d *Downloader) fetcher.ProtocolHost {
|
||||||
|
return fetcher.ProtocolHost{UpdateState: func(change func(config, state json.RawMessage) (any, error)) error {
|
||||||
|
return d.updateProtocolState("counter", change)
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProtocolUpdateStateCommitsOnlyStateAndUsesLatestConfig(t *testing.T) {
|
||||||
|
d := protocolConfigDownloader(t, nil, &counterManager{})
|
||||||
|
host := counterHost(d)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < 20; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
if err := fetcher.UpdateState[counterConfig, counterState](host, func(cfg *counterConfig, state *counterState) error { cfg.Writable = 999; state.Count++; return nil }); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
actual := protocolObject(t, d, "counter")
|
||||||
|
if actual["writable"] != float64(7) || actual["state"].(map[string]any)["count"] != float64(20) {
|
||||||
|
t.Fatal(actual)
|
||||||
|
}
|
||||||
|
// Rejecting a callback must not publish its partial mutation.
|
||||||
|
err := fetcher.UpdateState[counterConfig, counterState](host, func(_ *counterConfig, state *counterState) error { state.Count = 1000; return errors.New("reject") })
|
||||||
|
if err == nil || !reflect.DeepEqual(protocolObject(t, d, "counter"), actual) {
|
||||||
|
t.Fatal("failed callback published state")
|
||||||
|
}
|
||||||
|
if err := d.updateProtocolState("counter", func(_, _ json.RawMessage) (any, error) { return map[string]any{"count": "bad"}, nil }); err == nil {
|
||||||
|
t.Fatal("invalid state accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProtocolStateAndFullPutDoNotLoseConcurrentUpdates(t *testing.T) {
|
||||||
|
d := protocolConfigDownloader(t, nil, &counterManager{})
|
||||||
|
snapshot, _ := d.GetConfig()
|
||||||
|
snapshot.ProtocolConfig["counter"] = map[string]any{"writable": 21}
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < 20; i++ {
|
||||||
|
wg.Add(2)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
if err := d.PutConfig(snapshot); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
if err := fetcher.UpdateState[counterConfig, counterState](counterHost(d), func(cfg *counterConfig, state *counterState) error { state.Count++; return nil }); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
actual := protocolObject(t, d, "counter")
|
||||||
|
if actual["writable"] != float64(21) || actual["state"].(map[string]any)["count"] != float64(20) {
|
||||||
|
t.Fatal(actual)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBTSubscriptionStatusesSurviveStorageRestart(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
storage := NewBoltStorage(dir)
|
||||||
|
d := protocolConfigDownloader(t, storage, &btProtocol.FetcherManager{})
|
||||||
|
configureBT(t, d, "https://example.com/list")
|
||||||
|
seedBTState(t, d)
|
||||||
|
expected := protocolObject(t, d, "bt")["state"]
|
||||||
|
if err := storage.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reopened := NewBoltStorage(dir)
|
||||||
|
if err := reopened.Setup([]string{bucketConfig}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer reopened.Close()
|
||||||
|
var persisted base.DownloaderStoreConfig
|
||||||
|
if exists, err := reopened.Get(bucketConfig, "config", &persisted); err != nil || !exists {
|
||||||
|
t.Fatalf("reload persisted config: %v %v", exists, err)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(persisted.ProtocolConfig["bt"])
|
||||||
|
var cfg map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(cfg["state"], expected) {
|
||||||
|
t.Fatalf("state lost after reopening storage: %+v", cfg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type rejectedConfigStorage struct{ Storage }
|
||||||
|
|
||||||
|
func (*rejectedConfigStorage) Put(string, string, any) error {
|
||||||
|
return errors.New("storage unavailable")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProtocolStateAndPutStorageFailureDoNotPublish(t *testing.T) {
|
||||||
|
d := protocolConfigDownloader(t, nil, &counterManager{})
|
||||||
|
before := protocolObject(t, d, "counter")
|
||||||
|
d.storage = &rejectedConfigStorage{Storage: d.storage}
|
||||||
|
err := fetcher.UpdateState[counterConfig, counterState](counterHost(d), func(_ *counterConfig, state *counterState) error { state.Count = 99; return nil })
|
||||||
|
if err == nil || !reflect.DeepEqual(protocolObject(t, d, "counter"), before) {
|
||||||
|
t.Fatal("failed state persistence published a change")
|
||||||
|
}
|
||||||
|
cfg, _ := d.GetConfig()
|
||||||
|
cfg.ProtocolConfig["counter"] = map[string]any{"writable": 99}
|
||||||
|
if err := d.PutConfig(cfg); err == nil || !reflect.DeepEqual(protocolObject(t, d, "counter"), before) {
|
||||||
|
t.Fatal("failed config persistence published a change")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -191,13 +191,16 @@ func ensureScriptExecutable(t *testing.T, scriptPath string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func setupScriptTest(t *testing.T, fn func(downloader *Downloader)) {
|
func setupScriptTest(t *testing.T, fn func(downloader *Downloader)) {
|
||||||
defaultDownloader.Setup()
|
t.Helper()
|
||||||
defaultDownloader.cfg.StorageDir = ".test_storage"
|
// Each test owns its runtime; a closed Downloader cannot be reused.
|
||||||
defaultDownloader.cfg.DownloadDir = ".test_download"
|
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||||
defer func() {
|
if err := d.Setup(); err != nil {
|
||||||
defaultDownloader.Clear()
|
t.Fatal(err)
|
||||||
os.RemoveAll(defaultDownloader.cfg.StorageDir)
|
}
|
||||||
os.RemoveAll(defaultDownloader.cfg.DownloadDir)
|
t.Cleanup(func() {
|
||||||
}()
|
if err := d.Close(); err != nil {
|
||||||
fn(defaultDownloader)
|
t.Error(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
fn(d)
|
||||||
}
|
}
|
||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -858,13 +857,16 @@ func TestWebhook_TriggerOnPause(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func setupWebhookTest(t *testing.T, fn func(downloader *Downloader)) {
|
func setupWebhookTest(t *testing.T, fn func(downloader *Downloader)) {
|
||||||
defaultDownloader.Setup()
|
t.Helper()
|
||||||
defaultDownloader.cfg.StorageDir = ".test_storage"
|
// Each test owns its runtime; a closed Downloader cannot be reused.
|
||||||
defaultDownloader.cfg.DownloadDir = ".test_download"
|
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||||
defer func() {
|
if err := d.Setup(); err != nil {
|
||||||
defaultDownloader.Clear()
|
t.Fatal(err)
|
||||||
os.RemoveAll(defaultDownloader.cfg.StorageDir)
|
}
|
||||||
os.RemoveAll(defaultDownloader.cfg.DownloadDir)
|
t.Cleanup(func() {
|
||||||
}()
|
if err := d.Close(); err != nil {
|
||||||
fn(defaultDownloader)
|
t.Error(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
fn(d)
|
||||||
}
|
}
|
||||||
@@ -1,9 +1,7 @@
|
|||||||
package rest
|
package rest
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"io"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
|
||||||
"runtime"
|
"runtime"
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
@@ -337,43 +335,6 @@ func UpdateExtension(w http.ResponseWriter, r *http.Request) {
|
|||||||
WriteJson(w, model.NewNilResult())
|
WriteJson(w, model.NewNilResult())
|
||||||
}
|
}
|
||||||
|
|
||||||
func DoProxy(w http.ResponseWriter, r *http.Request) {
|
|
||||||
target := r.Header.Get("X-Target-Uri")
|
|
||||||
if target == "" {
|
|
||||||
writeError(w, "param invalid: X-Target-Uri")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
targetUrl, err := url.Parse(target)
|
|
||||||
if err != nil {
|
|
||||||
writeError(w, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
r.RequestURI = ""
|
|
||||||
r.URL = targetUrl
|
|
||||||
r.Host = targetUrl.Host
|
|
||||||
r.Header.Del("Cookie")
|
|
||||||
r.Header.Del("X-Api-Token")
|
|
||||||
r.Header.Del("X-Target-Uri")
|
|
||||||
resp, err := http.DefaultClient.Do(r)
|
|
||||||
if err != nil {
|
|
||||||
writeError(w, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
for k, vv := range resp.Header {
|
|
||||||
for _, v := range vv {
|
|
||||||
w.Header().Set(k, v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w.WriteHeader(resp.StatusCode)
|
|
||||||
buf, err := io.ReadAll(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
writeError(w, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.Write(buf)
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetStats(w http.ResponseWriter, r *http.Request) {
|
func GetStats(w http.ResponseWriter, r *http.Request) {
|
||||||
vars := mux.Vars(r)
|
vars := mux.Vars(r)
|
||||||
taskId := vars["id"]
|
taskId := vars["id"]
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package model
|
||||||
|
|
||||||
|
type MirrorCandidatesReq struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type MirrorCandidatesResp struct {
|
||||||
|
URLs []string `json:"urls"`
|
||||||
|
}
|
||||||
+31
-28
@@ -216,7 +216,6 @@ func buildServer(startCfg *model.StartConfig, continueOnStartup bool) (*http.Ser
|
|||||||
WriteStatusJson(w, response.StatusCode, response.Body)
|
WriteStatusJson(w, response.StatusCode, response.Body)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
r.Path("/api/web/proxy").HandlerFunc(DoProxy)
|
|
||||||
if startCfg.MCPEnable {
|
if startCfg.MCPEnable {
|
||||||
r.Path("/mcp").Handler(mcpserver.NewHandler(Downloader))
|
r.Path("/mcp").Handler(mcpserver.NewHandler(Downloader))
|
||||||
} else {
|
} else {
|
||||||
@@ -231,35 +230,39 @@ func buildServer(startCfg *model.StartConfig, continueOnStartup bool) (*http.Ser
|
|||||||
if enableWebAuth {
|
if enableWebAuth {
|
||||||
webSessions = newWebSessionStore()
|
webSessions = newWebSessionStore()
|
||||||
}
|
}
|
||||||
if startCfg.WebEnable {
|
if enableWebAuth {
|
||||||
if enableWebAuth {
|
r.Methods(http.MethodPost).Path("/api/web/login").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
r.Methods(http.MethodPost).Path("/api/web/login").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
var loginReq model.WebAuth
|
||||||
var loginReq model.WebAuth
|
if ReadJson(r, w, &loginReq) {
|
||||||
if ReadJson(r, w, &loginReq) {
|
if loginReq.Username == startCfg.WebAuth.Username && loginReq.Password == startCfg.WebAuth.Password {
|
||||||
if loginReq.Username == startCfg.WebAuth.Username && loginReq.Password == startCfg.WebAuth.Password {
|
sessionID, expiresAt, err := webSessions.create()
|
||||||
sessionID, expiresAt, err := webSessions.create()
|
if err != nil {
|
||||||
if err != nil {
|
WriteJson(w, model.NewErrorResult(err.Error()))
|
||||||
WriteJson(w, model.NewErrorResult(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
http.SetCookie(w, &http.Cookie{
|
|
||||||
Name: webAuthCookieName,
|
|
||||||
Value: sessionID,
|
|
||||||
Path: "/",
|
|
||||||
MaxAge: int(webAuthSessionTTL.Seconds()),
|
|
||||||
Expires: expiresAt,
|
|
||||||
HttpOnly: true,
|
|
||||||
Secure: requestUsesHTTPS(r),
|
|
||||||
SameSite: http.SameSiteLaxMode,
|
|
||||||
})
|
|
||||||
|
|
||||||
WriteJson(w, model.NewNilResult())
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
http.SetCookie(w, &http.Cookie{
|
||||||
|
Name: webAuthCookieName,
|
||||||
|
Value: sessionID,
|
||||||
|
Path: "/",
|
||||||
|
MaxAge: int(webAuthSessionTTL.Seconds()),
|
||||||
|
Expires: expiresAt,
|
||||||
|
HttpOnly: true,
|
||||||
|
Secure: requestUsesHTTPS(r),
|
||||||
|
SameSite: http.SameSiteLaxMode,
|
||||||
|
})
|
||||||
|
|
||||||
|
WriteJson(w, model.NewNilResult())
|
||||||
|
return
|
||||||
}
|
}
|
||||||
WriteStatusJson(w, http.StatusUnauthorized, model.NewErrorResult("unauthorized", model.CodeUnauthorized))
|
}
|
||||||
})
|
WriteStatusJson(w, http.StatusUnauthorized, model.NewErrorResult("unauthorized", model.CodeUnauthorized))
|
||||||
}
|
})
|
||||||
|
}
|
||||||
|
// Reserve the API namespace before the web file handler, including unknown
|
||||||
|
// endpoints and unsupported methods on otherwise known API paths.
|
||||||
|
r.Path("/api").Handler(http.NotFoundHandler())
|
||||||
|
r.PathPrefix("/api/").Handler(http.NotFoundHandler())
|
||||||
|
if startCfg.WebEnable {
|
||||||
r.PathPrefix("/fs/tasks").Handler(http.FileServer(new(taskFileSystem)))
|
r.PathPrefix("/fs/tasks").Handler(http.FileServer(new(taskFileSystem)))
|
||||||
r.PathPrefix("/fs/extensions").Handler(http.FileServer(new(extensionFileSystem)))
|
r.PathPrefix("/fs/extensions").Handler(http.FileServer(new(extensionFileSystem)))
|
||||||
r.PathPrefix("/").Handler(gzipMiddleware(http.FileServer(newEmbedCacheFileSystem(http.FS(startCfg.WebFS)))))
|
r.PathPrefix("/").Handler(gzipMiddleware(http.FileServer(newEmbedCacheFileSystem(http.FS(startCfg.WebFS)))))
|
||||||
@@ -325,7 +328,7 @@ func buildServer(startCfg *model.StartConfig, continueOnStartup bool) (*http.Ser
|
|||||||
})
|
})
|
||||||
|
|
||||||
server := &http.Server{Handler: handlers.CORS(
|
server := &http.Server{Handler: handlers.CORS(
|
||||||
handlers.AllowedHeaders([]string{"Accept", "Content-Type", "Authorization", "X-Api-Token", "X-Target-Uri", "Mcp-Method", "Mcp-Name", "Mcp-Protocol-Version", "Mcp-Session-Id", "Last-Event-ID"}),
|
handlers.AllowedHeaders([]string{"Accept", "Content-Type", "Authorization", "X-Api-Token", "Mcp-Method", "Mcp-Name", "Mcp-Protocol-Version", "Mcp-Session-Id", "Last-Event-ID"}),
|
||||||
handlers.AllowedMethods([]string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}),
|
handlers.AllowedMethods([]string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}),
|
||||||
handlers.AllowedOrigins([]string{"*"}),
|
handlers.AllowedOrigins([]string{"*"}),
|
||||||
handlers.ExposedHeaders([]string{"Mcp-Session-Id"}),
|
handlers.ExposedHeaders([]string{"Mcp-Session-Id"}),
|
||||||
|
|||||||
+44
-24
@@ -2,7 +2,6 @@ package rest
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"crypto/md5"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -15,6 +14,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/test"
|
"github.com/GopeedLab/gopeed/internal/test"
|
||||||
@@ -669,29 +669,49 @@ func TestWebFsEnhance(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDoProxy(t *testing.T) {
|
func TestUnknownAPIPathsReturn404(t *testing.T) {
|
||||||
doTest(func() {
|
for _, webEnabled := range []bool{false, true} {
|
||||||
code, respBody := doHttpRequest0(http.MethodGet, "/api/web/proxy", map[string]string{
|
t.Run(fmt.Sprintf("web=%t", webEnabled), func(t *testing.T) {
|
||||||
"X-Target-Uri": "https://github.com/GopeedLab/gopeed/raw/695da7ea87d2b455552b709d3cb4d7879484d4d1/README.md",
|
doTest0(func(cfg *model.StartConfig) {
|
||||||
}, nil)
|
cfg.WebEnable = webEnabled
|
||||||
if code != http.StatusOK {
|
cfg.WebFS = fstest.MapFS{
|
||||||
t.Errorf("DoProxy() got = %v, want %v", code, http.StatusOK)
|
"index.html": &fstest.MapFile{Data: []byte("<html>app</html>")},
|
||||||
}
|
"api/static.txt": &fstest.MapFile{Data: []byte("must not serve API paths as files")},
|
||||||
want := "4ee193b676f1ebb2ad810e016350d52a"
|
"apiary": &fstest.MapFile{Data: []byte("namespace boundary")},
|
||||||
got := fmt.Sprintf("%x", md5.Sum(respBody))
|
}
|
||||||
if got != want {
|
}, func() {
|
||||||
t.Errorf("DoProxy() got = %v, want %v", got, want)
|
paths := []string{"/api", "/api/", "/api/missing", "/api/v1/missing", "/api/web/missing", "/api/web/login", "/api/static.txt"}
|
||||||
}
|
for _, path := range paths {
|
||||||
})
|
for _, method := range []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete, http.MethodHead} {
|
||||||
|
code, body := doHttpRequest0(method, path, nil, nil)
|
||||||
doTest(func() {
|
if code != http.StatusNotFound {
|
||||||
code, _ := doHttpRequest0(http.MethodGet, "/api/web/proxy", map[string]string{
|
t.Fatalf("%s %s status %d, want 404: %s", method, path, code, body)
|
||||||
"X-Target-Uri": "https://github.com/GopeedLab/gopeed/raw/695da7ea87d2b455552b709d3cb4d7879484d4d1/NOT_FOUND",
|
}
|
||||||
}, nil)
|
if strings.Contains(string(body), "<html>app</html>") {
|
||||||
if code != http.StatusNotFound {
|
t.Fatalf("%s %s fell through to web files", method, path)
|
||||||
t.Errorf("DoProxy() got = %v, want %v", code, http.StatusNotFound)
|
}
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
|
// Known API routes still run, while unsupported methods use the API fallback.
|
||||||
|
code, _ := doHttpRequest0(http.MethodGet, "/api/v1/info", nil, nil)
|
||||||
|
if code != http.StatusOK {
|
||||||
|
t.Fatalf("known API status %d, want 200", code)
|
||||||
|
}
|
||||||
|
code, _ = doHttpRequest0(http.MethodPost, "/api/v1/info", nil, nil)
|
||||||
|
if code != http.StatusNotFound {
|
||||||
|
t.Fatalf("unsupported API method status %d, want 404", code)
|
||||||
|
}
|
||||||
|
if webEnabled {
|
||||||
|
for path, want := range map[string]string{"/": "<html>app</html>", "/apiary": "namespace boundary"} {
|
||||||
|
code, body := doHttpRequest0(http.MethodGet, path, map[string]string{"Accept-Encoding": "identity"}, nil)
|
||||||
|
if code != http.StatusOK || !strings.Contains(string(body), want) {
|
||||||
|
t.Fatalf("web file %s status %d body %s", path, code, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTestWebhook(t *testing.T) {
|
func TestTestWebhook(t *testing.T) {
|
||||||
|
|||||||
@@ -68,12 +68,11 @@ The following endpoints are web HTTP infrastructure and are not part of the FFI/
|
|||||||
|
|
||||||
- `/fs/tasks/*`
|
- `/fs/tasks/*`
|
||||||
- `/fs/extensions/*`
|
- `/fs/extensions/*`
|
||||||
- `/api/web/proxy`
|
|
||||||
- Web static file hosting at `/`
|
- Web static file hosting at `/`
|
||||||
|
|
||||||
`/fs/tasks` returns files from the Go host machine to a browser over HTTP. Native clients can access local file paths directly and do not need an equivalent API.
|
`/fs/tasks` returns files from the Go host machine to a browser over HTTP. Native clients can access local file paths directly and do not need an equivalent API.
|
||||||
|
|
||||||
`/api/web/proxy` is infrastructure used by the web transport to work around browser CORS restrictions. It is not part of the published core `/api/v1/**` API. Native clients can request the target URL directly with Dio and do not need the Go proxy.
|
All unmatched `/api` and `/api/**` requests use the common 404 handler before the web file handler.
|
||||||
|
|
||||||
## 4. Backend Maintenance Model
|
## 4. Backend Maintenance Model
|
||||||
|
|
||||||
@@ -182,7 +181,7 @@ The REST adapter is limited to transport responsibilities:
|
|||||||
- Convert `http.Request` into a Dispatcher request.
|
- Convert `http.Request` into a Dispatcher request.
|
||||||
- Write a Dispatcher response back to HTTP.
|
- Write a Dispatcher response back to HTTP.
|
||||||
- Handle API tokens, web login, CORS, compression, and panic recovery.
|
- Handle API tokens, web login, CORS, compression, and panic recovery.
|
||||||
- Register web-only file, proxy, and static resource routes.
|
- Register web-only file and static resource routes.
|
||||||
|
|
||||||
The REST adapter no longer calls the global `rest.Downloader` directly.
|
The REST adapter no longer calls the global `rest.Downloader` directly.
|
||||||
|
|
||||||
@@ -198,7 +197,6 @@ Core API routes
|
|||||||
|
|
||||||
Web-only routes
|
Web-only routes
|
||||||
├── /api/web/login
|
├── /api/web/login
|
||||||
├── /api/web/proxy
|
|
||||||
├── /fs/tasks/*
|
├── /fs/tasks/*
|
||||||
├── /fs/extensions/*
|
├── /fs/extensions/*
|
||||||
└── /* static Web UI
|
└── /* static Web UI
|
||||||
@@ -416,12 +414,7 @@ The following patterns are prohibited:
|
|||||||
|
|
||||||
### 7.4 Non-business HTTP capabilities
|
### 7.4 Non-business HTTP capabilities
|
||||||
|
|
||||||
`proxyRequest` is not a Gopeed Core API and belongs in a separate `ExternalHttpClient`:
|
Tracker subscriptions run in Go. Application update checks and extension store requests run directly in the client. Go supplies mirror candidate URLs through `/api/v1/mirrors/{provider}/candidates`; clients probe and download update packages locally with Dio to retain real download progress. There is no arbitrary external HTTP forwarding API.
|
||||||
|
|
||||||
```text
|
|
||||||
Native → request the target directly with Dio
|
|
||||||
Web → /api/web/proxy
|
|
||||||
```
|
|
||||||
|
|
||||||
Task file access does not belong in `GopeedClient`:
|
Task file access does not belong in `GopeedClient`:
|
||||||
|
|
||||||
@@ -594,7 +587,7 @@ Flutter converts all of them into `GopeedException`. Components do not handle Di
|
|||||||
|
|
||||||
- Windows, macOS, Linux: startup, InvokeAsync, events, shutdown, and repeated startup.
|
- Windows, macOS, Linux: startup, InvokeAsync, events, shutdown, and repeated startup.
|
||||||
- Android, iOS: gomobile InvokeAsync callbacks, reverse MethodChannel events, and background/foreground transitions.
|
- Android, iOS: gomobile InvokeAsync callbacks, reverse MethodChannel events, and background/foreground transitions.
|
||||||
- Web: REST, login, proxy, and task-file downloads.
|
- Web: REST, login, and task-file downloads.
|
||||||
- Native: enabling and disabling REST does not interrupt download tasks.
|
- Native: enabling and disabling REST does not interrupt download tasks.
|
||||||
|
|
||||||
## 12. Implementation Phases
|
## 12. Implementation Phases
|
||||||
|
|||||||
@@ -208,11 +208,6 @@ Future<void> login(LoginReq loginReq) {
|
|||||||
return _parse<void>(() => _request('api/web/login', method: 'POST', data: loginReq.toJson()), null);
|
return _parse<void>(() => _request('api/web/login', method: 'POST', data: loginReq.toJson()), null);
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<Response<String>> proxyRequest<T>(String uri, {dynamic data, Options? options}) async {
|
|
||||||
initDefault();
|
|
||||||
return _transport!.proxyRequest(uri, data: data, options: options);
|
|
||||||
}
|
|
||||||
|
|
||||||
String join(String path) {
|
String join(String path) {
|
||||||
initDefault();
|
initDefault();
|
||||||
return _transport!.join(path);
|
return _transport!.join(path);
|
||||||
@@ -226,3 +221,16 @@ Future<dynamic> forward(
|
|||||||
}) async {
|
}) async {
|
||||||
return _request(path, method: method, data: data, queryParameters: queryParameters);
|
return _request(path, method: method, data: data, queryParameters: queryParameters);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Future<void> triggerProtocolJob(String protocol, String name) => _parse<void>(
|
||||||
|
() => _request(
|
||||||
|
'api/v1/protocols/${Uri.encodeComponent(protocol)}/jobs/${Uri.encodeComponent(name)}/trigger',
|
||||||
|
method: 'POST',
|
||||||
|
),
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
|
||||||
|
Future<List<String>> getMirrorCandidates(String provider, String url) => _parse(
|
||||||
|
() => _request('api/v1/mirrors/${Uri.encodeComponent(provider)}/candidates', method: 'POST', data: {'url': url}),
|
||||||
|
(json) => List<String>.from(json['urls'] as List),
|
||||||
|
);
|
||||||
@@ -1,27 +1,18 @@
|
|||||||
import 'dart:convert';
|
|
||||||
|
|
||||||
import 'package:dio/dio.dart';
|
import 'package:dio/dio.dart';
|
||||||
|
|
||||||
import 'api.dart';
|
|
||||||
import 'model/store_extension.dart';
|
import 'model/store_extension.dart';
|
||||||
|
|
||||||
class GopeedSiteApi {
|
class GopeedSiteApi {
|
||||||
GopeedSiteApi._();
|
GopeedSiteApi({Dio? client})
|
||||||
|
: _client =
|
||||||
|
client ??
|
||||||
|
Dio(BaseOptions(connectTimeout: const Duration(seconds: 10), receiveTimeout: const Duration(seconds: 10)));
|
||||||
|
|
||||||
static final instance = GopeedSiteApi._();
|
static final instance = GopeedSiteApi();
|
||||||
|
final Dio _client;
|
||||||
static const _host = 'gopeed.com';
|
|
||||||
final _storePages = <String, ({StoreExtensionPage page, DateTime expires})>{};
|
final _storePages = <String, ({StoreExtensionPage page, DateTime expires})>{};
|
||||||
final _storeRequests = <String, Future<StoreExtensionPage>>{};
|
final _storeRequests = <String, Future<StoreExtensionPage>>{};
|
||||||
|
|
||||||
Future<List<dynamic>> getReleases({int perPage = 10}) async {
|
|
||||||
final json = await _getJson('/api/releases', queryParameters: {'per_page': perPage.clamp(1, 100).toString()});
|
|
||||||
if (json is! List<dynamic>) {
|
|
||||||
throw const FormatException('Invalid releases response');
|
|
||||||
}
|
|
||||||
return json;
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<StoreExtensionPage> getExtensions({
|
Future<StoreExtensionPage> getExtensions({
|
||||||
int page = 1,
|
int page = 1,
|
||||||
int limit = 20,
|
int limit = 20,
|
||||||
@@ -31,14 +22,14 @@ class GopeedSiteApi {
|
|||||||
bool forceRefresh = false,
|
bool forceRefresh = false,
|
||||||
}) async {
|
}) async {
|
||||||
final params = {
|
final params = {
|
||||||
|
'view': 'summary',
|
||||||
'page': page.toString(),
|
'page': page.toString(),
|
||||||
'limit': limit.clamp(1, 100).toString(),
|
'limit': limit.clamp(1, 100).toString(),
|
||||||
'sort': sort.name,
|
'sort': sort.name,
|
||||||
'order': order.name,
|
'order': order.name,
|
||||||
'view': 'summary',
|
|
||||||
if (query != null && query.trim().isNotEmpty) 'q': query.trim(),
|
if (query != null && query.trim().isNotEmpty) 'q': query.trim(),
|
||||||
};
|
};
|
||||||
final key = Uri.https(_host, '/api/extensions', params).toString();
|
final key = Uri(queryParameters: params).toString();
|
||||||
final now = DateTime.now();
|
final now = DateTime.now();
|
||||||
_storePages.removeWhere((_, entry) => !now.isBefore(entry.expires));
|
_storePages.removeWhere((_, entry) => !now.isBefore(entry.expires));
|
||||||
final cached = _storePages[key];
|
final cached = _storePages[key];
|
||||||
@@ -58,40 +49,34 @@ class GopeedSiteApi {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Future<StoreExtensionPage> _fetchStorePage(Map<String, String> params) async {
|
Future<StoreExtensionPage> _fetchStorePage(Map<String, String> params) async {
|
||||||
final json = await _getJson('/api/extensions', queryParameters: params);
|
final response = await _client.get<Map<String, dynamic>>(
|
||||||
return StoreExtensionPage.fromJson(json as Map<String, dynamic>);
|
'https://gopeed.com/api/extensions',
|
||||||
|
queryParameters: params,
|
||||||
|
);
|
||||||
|
return StoreExtensionPage.fromJson(response.data!);
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<StoreExtension> getExtension(String id, {String? version}) async {
|
Future<StoreExtension> getExtension(String id, {String? version}) async {
|
||||||
final json = await _getJsonUri(
|
final uri = Uri(
|
||||||
Uri(
|
scheme: 'https',
|
||||||
scheme: 'https',
|
host: 'gopeed.com',
|
||||||
host: _host,
|
pathSegments: ['api', 'extensions', id],
|
||||||
pathSegments: ['api', 'extensions', id],
|
queryParameters: {'version': ?version},
|
||||||
queryParameters: {if (version != null && version.isNotEmpty) 'version': version},
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
return StoreExtension.fromJson(json as Map<String, dynamic>);
|
final response = await _client.get<Map<String, dynamic>>(uri.toString());
|
||||||
|
return StoreExtension.fromJson(response.data!);
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<dynamic> getReleases() async {
|
||||||
|
final response = await _client.get<dynamic>(
|
||||||
|
'https://gopeed.com/api/releases',
|
||||||
|
queryParameters: {'per_page': 10},
|
||||||
|
options: Options(followRedirects: false),
|
||||||
|
);
|
||||||
|
return response.data;
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<void> reportExtensionInstall(String id) async {
|
Future<void> reportExtensionInstall(String id) async {
|
||||||
final uri = Uri.https(_host, '/api/extensions/install');
|
await _client.post<void>('https://gopeed.com/api/extensions/install', data: {'id': id});
|
||||||
await proxyRequest(
|
|
||||||
uri.toString(),
|
|
||||||
data: {'id': id},
|
|
||||||
options: Options(method: 'POST', contentType: Headers.jsonContentType),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<dynamic> _getJson(String path, {Map<String, String>? queryParameters}) {
|
|
||||||
return _getJsonUri(Uri.https(_host, path, queryParameters));
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<dynamic> _getJsonUri(Uri uri) async {
|
|
||||||
final Response<String> response = await proxyRequest(uri.toString());
|
|
||||||
if (response.data == null || response.data!.isEmpty) {
|
|
||||||
throw Exception('Empty response from $uri');
|
|
||||||
}
|
|
||||||
return jsonDecode(response.data!);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -8,6 +8,7 @@ class DownloaderConfig {
|
|||||||
int maxRunning;
|
int maxRunning;
|
||||||
ProtocolConfig protocolConfig = ProtocolConfig();
|
ProtocolConfig protocolConfig = ProtocolConfig();
|
||||||
ExtraConfig extra = ExtraConfig();
|
ExtraConfig extra = ExtraConfig();
|
||||||
|
GithubMirrorConfig githubMirror = GithubMirrorConfig();
|
||||||
ProxyConfig proxy = ProxyConfig();
|
ProxyConfig proxy = ProxyConfig();
|
||||||
WebhookConfig webhook = WebhookConfig();
|
WebhookConfig webhook = WebhookConfig();
|
||||||
ScriptConfig script = ScriptConfig();
|
ScriptConfig script = ScriptConfig();
|
||||||
@@ -98,10 +99,43 @@ class HttpConfig {
|
|||||||
Map<String, dynamic> toJson() => _$HttpConfigToJson(this);
|
Map<String, dynamic> toJson() => _$HttpConfigToJson(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DateTime? _trackerUpdateTimeFromJson(String? value) =>
|
||||||
|
value == null || value.startsWith('0001-') ? null : DateTime.parse(value);
|
||||||
|
|
||||||
@JsonSerializable()
|
@JsonSerializable()
|
||||||
|
class TrackerSubscriptionStatus {
|
||||||
|
final bool success;
|
||||||
|
final int trackerCount;
|
||||||
|
final DateTime lastFetchTime;
|
||||||
|
|
||||||
|
TrackerSubscriptionStatus({required this.success, required this.trackerCount, required this.lastFetchTime});
|
||||||
|
|
||||||
|
factory TrackerSubscriptionStatus.fromJson(Map<String, dynamic> json) => _$TrackerSubscriptionStatusFromJson(json);
|
||||||
|
|
||||||
|
Map<String, dynamic> toJson() => _$TrackerSubscriptionStatusToJson(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonSerializable(explicitToJson: true)
|
||||||
|
class BtState {
|
||||||
|
Map<String, List<String>> trackerSources;
|
||||||
|
Map<String, TrackerSubscriptionStatus> trackerSubscriptionStatuses;
|
||||||
|
@JsonKey(fromJson: _trackerUpdateTimeFromJson)
|
||||||
|
DateTime? lastTrackerUpdateTime;
|
||||||
|
|
||||||
|
BtState({this.trackerSources = const {}, this.trackerSubscriptionStatuses = const {}, this.lastTrackerUpdateTime});
|
||||||
|
|
||||||
|
factory BtState.fromJson(Map<String, dynamic>? json) => json == null ? BtState() : _$BtStateFromJson(json);
|
||||||
|
|
||||||
|
Map<String, dynamic> toJson() => _$BtStateToJson(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonSerializable(explicitToJson: true)
|
||||||
class BtConfig {
|
class BtConfig {
|
||||||
|
BtState state = BtState();
|
||||||
|
List<String> trackerSubscribeUrls;
|
||||||
|
bool autoUpdateTrackers;
|
||||||
|
List<String> customTrackers;
|
||||||
int listenPort;
|
int listenPort;
|
||||||
List<String> trackers;
|
|
||||||
bool seedKeep;
|
bool seedKeep;
|
||||||
double seedRatio;
|
double seedRatio;
|
||||||
int seedTime;
|
int seedTime;
|
||||||
@@ -109,8 +143,10 @@ class BtConfig {
|
|||||||
bool disablePex;
|
bool disablePex;
|
||||||
|
|
||||||
BtConfig({
|
BtConfig({
|
||||||
|
this.trackerSubscribeUrls = const [],
|
||||||
|
this.autoUpdateTrackers = true,
|
||||||
|
this.customTrackers = const [],
|
||||||
this.listenPort = 0,
|
this.listenPort = 0,
|
||||||
this.trackers = const [],
|
|
||||||
this.seedKeep = false,
|
this.seedKeep = false,
|
||||||
this.seedRatio = 0,
|
this.seedRatio = 0,
|
||||||
this.seedTime = 0,
|
this.seedTime = 0,
|
||||||
@@ -158,9 +194,6 @@ class ExtraConfig {
|
|||||||
String analyticsClientId;
|
String analyticsClientId;
|
||||||
List<DownloadCategory> downloadCategories;
|
List<DownloadCategory> downloadCategories;
|
||||||
|
|
||||||
ExtraConfigBt bt = ExtraConfigBt();
|
|
||||||
ExtraConfigGithubMirror githubMirror = ExtraConfigGithubMirror();
|
|
||||||
|
|
||||||
ExtraConfig({
|
ExtraConfig({
|
||||||
this.themeMode = '',
|
this.themeMode = '',
|
||||||
this.themeColor = 'green',
|
this.themeColor = 'green',
|
||||||
@@ -271,22 +304,6 @@ class ProxyConfig {
|
|||||||
Map<String, dynamic> toJson() => _$ProxyConfigToJson(this);
|
Map<String, dynamic> toJson() => _$ProxyConfigToJson(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonSerializable()
|
|
||||||
class ExtraConfigBt {
|
|
||||||
List<String> trackerSubscribeUrls = [];
|
|
||||||
List<String> subscribeTrackers = [];
|
|
||||||
bool autoUpdateTrackers = true;
|
|
||||||
DateTime? lastTrackerUpdateTime;
|
|
||||||
|
|
||||||
List<String> customTrackers = [];
|
|
||||||
|
|
||||||
ExtraConfigBt();
|
|
||||||
|
|
||||||
factory ExtraConfigBt.fromJson(Map<String, dynamic> json) => _$ExtraConfigBtFromJson(json);
|
|
||||||
|
|
||||||
Map<String, dynamic> toJson() => _$ExtraConfigBtToJson(this);
|
|
||||||
}
|
|
||||||
|
|
||||||
enum GithubMirrorType { jsdelivr, ghProxy }
|
enum GithubMirrorType { jsdelivr, ghProxy }
|
||||||
|
|
||||||
@JsonSerializable()
|
@JsonSerializable()
|
||||||
@@ -304,16 +321,16 @@ class GithubMirror {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@JsonSerializable(explicitToJson: true)
|
@JsonSerializable(explicitToJson: true)
|
||||||
class ExtraConfigGithubMirror {
|
class GithubMirrorConfig {
|
||||||
bool enabled;
|
bool enabled;
|
||||||
List<GithubMirror> mirrors;
|
List<GithubMirror> mirrors;
|
||||||
|
|
||||||
ExtraConfigGithubMirror({this.enabled = true, this.mirrors = const []});
|
GithubMirrorConfig({this.enabled = true, this.mirrors = const []});
|
||||||
|
|
||||||
factory ExtraConfigGithubMirror.fromJson(Map<String, dynamic>? json) =>
|
factory GithubMirrorConfig.fromJson(Map<String, dynamic>? json) =>
|
||||||
json == null ? ExtraConfigGithubMirror() : _$ExtraConfigGithubMirrorFromJson(json);
|
json == null ? GithubMirrorConfig() : _$GithubMirrorConfigFromJson(json);
|
||||||
|
|
||||||
Map<String, dynamic> toJson() => _$ExtraConfigGithubMirrorToJson(this);
|
Map<String, dynamic> toJson() => _$GithubMirrorConfigToJson(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonSerializable()
|
@JsonSerializable()
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ DownloaderConfig _$DownloaderConfigFromJson(Map<String, dynamic> json) =>
|
|||||||
)
|
)
|
||||||
..protocolConfig = ProtocolConfig.fromJson(json['protocolConfig'] as Map<String, dynamic>?)
|
..protocolConfig = ProtocolConfig.fromJson(json['protocolConfig'] as Map<String, dynamic>?)
|
||||||
..extra = ExtraConfig.fromJson(json['extra'] as Map<String, dynamic>?)
|
..extra = ExtraConfig.fromJson(json['extra'] as Map<String, dynamic>?)
|
||||||
|
..githubMirror = GithubMirrorConfig.fromJson(json['githubMirror'] as Map<String, dynamic>?)
|
||||||
..proxy = ProxyConfig.fromJson(json['proxy'] as Map<String, dynamic>)
|
..proxy = ProxyConfig.fromJson(json['proxy'] as Map<String, dynamic>)
|
||||||
..webhook = WebhookConfig.fromJson(json['webhook'] as Map<String, dynamic>?)
|
..webhook = WebhookConfig.fromJson(json['webhook'] as Map<String, dynamic>?)
|
||||||
..script = ScriptConfig.fromJson(json['script'] as Map<String, dynamic>?)
|
..script = ScriptConfig.fromJson(json['script'] as Map<String, dynamic>?)
|
||||||
@@ -27,6 +28,7 @@ Map<String, dynamic> _$DownloaderConfigToJson(DownloaderConfig instance) => <Str
|
|||||||
'maxRunning': instance.maxRunning,
|
'maxRunning': instance.maxRunning,
|
||||||
'protocolConfig': instance.protocolConfig.toJson(),
|
'protocolConfig': instance.protocolConfig.toJson(),
|
||||||
'extra': instance.extra.toJson(),
|
'extra': instance.extra.toJson(),
|
||||||
|
'githubMirror': instance.githubMirror.toJson(),
|
||||||
'proxy': instance.proxy.toJson(),
|
'proxy': instance.proxy.toJson(),
|
||||||
'webhook': instance.webhook.toJson(),
|
'webhook': instance.webhook.toJson(),
|
||||||
'script': instance.script.toJson(),
|
'script': instance.script.toJson(),
|
||||||
@@ -92,19 +94,56 @@ Map<String, dynamic> _$HttpConfigToJson(HttpConfig instance) => <String, dynamic
|
|||||||
'useServerCtime': instance.useServerCtime,
|
'useServerCtime': instance.useServerCtime,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
TrackerSubscriptionStatus _$TrackerSubscriptionStatusFromJson(Map<String, dynamic> json) => TrackerSubscriptionStatus(
|
||||||
|
success: json['success'] as bool,
|
||||||
|
trackerCount: (json['trackerCount'] as num).toInt(),
|
||||||
|
lastFetchTime: DateTime.parse(json['lastFetchTime'] as String),
|
||||||
|
);
|
||||||
|
|
||||||
|
Map<String, dynamic> _$TrackerSubscriptionStatusToJson(TrackerSubscriptionStatus instance) => <String, dynamic>{
|
||||||
|
'success': instance.success,
|
||||||
|
'trackerCount': instance.trackerCount,
|
||||||
|
'lastFetchTime': instance.lastFetchTime.toIso8601String(),
|
||||||
|
};
|
||||||
|
|
||||||
|
BtState _$BtStateFromJson(Map<String, dynamic> json) => BtState(
|
||||||
|
trackerSources:
|
||||||
|
(json['trackerSources'] as Map<String, dynamic>?)?.map(
|
||||||
|
(k, e) => MapEntry(k, (e as List<dynamic>).map((e) => e as String).toList()),
|
||||||
|
) ??
|
||||||
|
const {},
|
||||||
|
trackerSubscriptionStatuses:
|
||||||
|
(json['trackerSubscriptionStatuses'] as Map<String, dynamic>?)?.map(
|
||||||
|
(k, e) => MapEntry(k, TrackerSubscriptionStatus.fromJson(e as Map<String, dynamic>)),
|
||||||
|
) ??
|
||||||
|
const {},
|
||||||
|
lastTrackerUpdateTime: _trackerUpdateTimeFromJson(json['lastTrackerUpdateTime'] as String?),
|
||||||
|
);
|
||||||
|
|
||||||
|
Map<String, dynamic> _$BtStateToJson(BtState instance) => <String, dynamic>{
|
||||||
|
'trackerSources': instance.trackerSources,
|
||||||
|
'trackerSubscriptionStatuses': instance.trackerSubscriptionStatuses.map((k, e) => MapEntry(k, e.toJson())),
|
||||||
|
'lastTrackerUpdateTime': ?instance.lastTrackerUpdateTime?.toIso8601String(),
|
||||||
|
};
|
||||||
|
|
||||||
BtConfig _$BtConfigFromJson(Map<String, dynamic> json) => BtConfig(
|
BtConfig _$BtConfigFromJson(Map<String, dynamic> json) => BtConfig(
|
||||||
|
trackerSubscribeUrls: (json['trackerSubscribeUrls'] as List<dynamic>?)?.map((e) => e as String).toList() ?? const [],
|
||||||
|
autoUpdateTrackers: json['autoUpdateTrackers'] as bool? ?? true,
|
||||||
|
customTrackers: (json['customTrackers'] as List<dynamic>?)?.map((e) => e as String).toList() ?? const [],
|
||||||
listenPort: (json['listenPort'] as num?)?.toInt() ?? 0,
|
listenPort: (json['listenPort'] as num?)?.toInt() ?? 0,
|
||||||
trackers: (json['trackers'] as List<dynamic>?)?.map((e) => e as String).toList() ?? const [],
|
|
||||||
seedKeep: json['seedKeep'] as bool? ?? false,
|
seedKeep: json['seedKeep'] as bool? ?? false,
|
||||||
seedRatio: (json['seedRatio'] as num?)?.toDouble() ?? 0,
|
seedRatio: (json['seedRatio'] as num?)?.toDouble() ?? 0,
|
||||||
seedTime: (json['seedTime'] as num?)?.toInt() ?? 0,
|
seedTime: (json['seedTime'] as num?)?.toInt() ?? 0,
|
||||||
disableDht: json['disableDht'] as bool? ?? false,
|
disableDht: json['disableDht'] as bool? ?? false,
|
||||||
disablePex: json['disablePex'] as bool? ?? false,
|
disablePex: json['disablePex'] as bool? ?? false,
|
||||||
);
|
)..state = BtState.fromJson(json['state'] as Map<String, dynamic>?);
|
||||||
|
|
||||||
Map<String, dynamic> _$BtConfigToJson(BtConfig instance) => <String, dynamic>{
|
Map<String, dynamic> _$BtConfigToJson(BtConfig instance) => <String, dynamic>{
|
||||||
|
'state': instance.state.toJson(),
|
||||||
|
'trackerSubscribeUrls': instance.trackerSubscribeUrls,
|
||||||
|
'autoUpdateTrackers': instance.autoUpdateTrackers,
|
||||||
|
'customTrackers': instance.customTrackers,
|
||||||
'listenPort': instance.listenPort,
|
'listenPort': instance.listenPort,
|
||||||
'trackers': instance.trackers,
|
|
||||||
'seedKeep': instance.seedKeep,
|
'seedKeep': instance.seedKeep,
|
||||||
'seedRatio': instance.seedRatio,
|
'seedRatio': instance.seedRatio,
|
||||||
'seedTime': instance.seedTime,
|
'seedTime': instance.seedTime,
|
||||||
@@ -128,37 +167,31 @@ Map<String, dynamic> _$Ed2kConfigToJson(Ed2kConfig instance) => <String, dynamic
|
|||||||
'nodesDat': instance.nodesDat,
|
'nodesDat': instance.nodesDat,
|
||||||
};
|
};
|
||||||
|
|
||||||
ExtraConfig _$ExtraConfigFromJson(Map<String, dynamic> json) =>
|
ExtraConfig _$ExtraConfigFromJson(Map<String, dynamic> json) => ExtraConfig(
|
||||||
ExtraConfig(
|
themeMode: json['themeMode'] as String? ?? '',
|
||||||
themeMode: json['themeMode'] as String? ?? '',
|
themeColor: json['themeColor'] as String? ?? 'green',
|
||||||
themeColor: json['themeColor'] as String? ?? 'green',
|
locale: json['locale'] as String? ?? '',
|
||||||
locale: json['locale'] as String? ?? '',
|
lastDeleteTaskKeep: json['lastDeleteTaskKeep'] as bool? ?? false,
|
||||||
lastDeleteTaskKeep: json['lastDeleteTaskKeep'] as bool? ?? false,
|
defaultDirectDownload: json['defaultDirectDownload'] as bool? ?? false,
|
||||||
defaultDirectDownload: json['defaultDirectDownload'] as bool? ?? false,
|
defaultBtClient: json['defaultBtClient'] as bool? ?? true,
|
||||||
defaultBtClient: json['defaultBtClient'] as bool? ?? true,
|
notifyWhenNewVersion: json['notifyWhenNewVersion'] as bool? ?? true,
|
||||||
notifyWhenNewVersion: json['notifyWhenNewVersion'] as bool? ?? true,
|
desktopNotification: json['desktopNotification'] as bool? ?? true,
|
||||||
desktopNotification: json['desktopNotification'] as bool? ?? true,
|
backgroundLocationKeepAlive: json['backgroundLocationKeepAlive'] as bool? ?? false,
|
||||||
backgroundLocationKeepAlive: json['backgroundLocationKeepAlive'] as bool? ?? false,
|
backgroundContinuedProcessing: json['backgroundContinuedProcessing'] as bool? ?? false,
|
||||||
backgroundContinuedProcessing:
|
windowState: json['windowState'] == null
|
||||||
json['backgroundContinuedProcessing']
|
? null
|
||||||
as bool? ??
|
: WindowStateConfig.fromJson(json['windowState'] as Map<String, dynamic>?),
|
||||||
false,
|
bookmarks: (json['bookmarks'] as Map<String, dynamic>?)?.map((k, e) => MapEntry(k, e as String)) ?? const {},
|
||||||
windowState: json['windowState'] == null
|
createHistory: (json['createHistory'] as List<dynamic>?)?.map((e) => e as String).toList() ?? const [],
|
||||||
? null
|
runAsMenubarApp: json['runAsMenubarApp'] as bool? ?? false,
|
||||||
: WindowStateConfig.fromJson(json['windowState'] as Map<String, dynamic>?),
|
analyticsEnabled: json['analyticsEnabled'] as bool? ?? true,
|
||||||
bookmarks: (json['bookmarks'] as Map<String, dynamic>?)?.map((k, e) => MapEntry(k, e as String)) ?? const {},
|
analyticsClientId: json['analyticsClientId'] as String? ?? '',
|
||||||
createHistory: (json['createHistory'] as List<dynamic>?)?.map((e) => e as String).toList() ?? const [],
|
downloadCategories:
|
||||||
runAsMenubarApp: json['runAsMenubarApp'] as bool? ?? false,
|
(json['downloadCategories'] as List<dynamic>?)
|
||||||
analyticsEnabled: json['analyticsEnabled'] as bool? ?? true,
|
?.map((e) => DownloadCategory.fromJson(e as Map<String, dynamic>))
|
||||||
analyticsClientId: json['analyticsClientId'] as String? ?? '',
|
.toList() ??
|
||||||
downloadCategories:
|
const [],
|
||||||
(json['downloadCategories'] as List<dynamic>?)
|
);
|
||||||
?.map((e) => DownloadCategory.fromJson(e as Map<String, dynamic>))
|
|
||||||
.toList() ??
|
|
||||||
const [],
|
|
||||||
)
|
|
||||||
..bt = ExtraConfigBt.fromJson(json['bt'] as Map<String, dynamic>)
|
|
||||||
..githubMirror = ExtraConfigGithubMirror.fromJson(json['githubMirror'] as Map<String, dynamic>?);
|
|
||||||
|
|
||||||
Map<String, dynamic> _$ExtraConfigToJson(ExtraConfig instance) => <String, dynamic>{
|
Map<String, dynamic> _$ExtraConfigToJson(ExtraConfig instance) => <String, dynamic>{
|
||||||
'themeMode': instance.themeMode,
|
'themeMode': instance.themeMode,
|
||||||
@@ -170,8 +203,7 @@ Map<String, dynamic> _$ExtraConfigToJson(ExtraConfig instance) => <String, dynam
|
|||||||
'notifyWhenNewVersion': instance.notifyWhenNewVersion,
|
'notifyWhenNewVersion': instance.notifyWhenNewVersion,
|
||||||
'desktopNotification': instance.desktopNotification,
|
'desktopNotification': instance.desktopNotification,
|
||||||
'backgroundLocationKeepAlive': instance.backgroundLocationKeepAlive,
|
'backgroundLocationKeepAlive': instance.backgroundLocationKeepAlive,
|
||||||
'backgroundContinuedProcessing':
|
'backgroundContinuedProcessing': instance.backgroundContinuedProcessing,
|
||||||
instance.backgroundContinuedProcessing,
|
|
||||||
'windowState': instance.windowState.toJson(),
|
'windowState': instance.windowState.toJson(),
|
||||||
'bookmarks': instance.bookmarks,
|
'bookmarks': instance.bookmarks,
|
||||||
'createHistory': instance.createHistory,
|
'createHistory': instance.createHistory,
|
||||||
@@ -179,8 +211,6 @@ Map<String, dynamic> _$ExtraConfigToJson(ExtraConfig instance) => <String, dynam
|
|||||||
'analyticsEnabled': instance.analyticsEnabled,
|
'analyticsEnabled': instance.analyticsEnabled,
|
||||||
'analyticsClientId': instance.analyticsClientId,
|
'analyticsClientId': instance.analyticsClientId,
|
||||||
'downloadCategories': instance.downloadCategories.map((e) => e.toJson()).toList(),
|
'downloadCategories': instance.downloadCategories.map((e) => e.toJson()).toList(),
|
||||||
'bt': instance.bt.toJson(),
|
|
||||||
'githubMirror': instance.githubMirror.toJson(),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
WindowStateConfig _$WindowStateConfigFromJson(Map<String, dynamic> json) => WindowStateConfig(
|
WindowStateConfig _$WindowStateConfigFromJson(Map<String, dynamic> json) => WindowStateConfig(
|
||||||
@@ -249,23 +279,6 @@ Map<String, dynamic> _$ProxyConfigToJson(ProxyConfig instance) => <String, dynam
|
|||||||
'pwd': instance.pwd,
|
'pwd': instance.pwd,
|
||||||
};
|
};
|
||||||
|
|
||||||
ExtraConfigBt _$ExtraConfigBtFromJson(Map<String, dynamic> json) => ExtraConfigBt()
|
|
||||||
..trackerSubscribeUrls = (json['trackerSubscribeUrls'] as List<dynamic>).map((e) => e as String).toList()
|
|
||||||
..subscribeTrackers = (json['subscribeTrackers'] as List<dynamic>).map((e) => e as String).toList()
|
|
||||||
..autoUpdateTrackers = json['autoUpdateTrackers'] as bool
|
|
||||||
..lastTrackerUpdateTime = json['lastTrackerUpdateTime'] == null
|
|
||||||
? null
|
|
||||||
: DateTime.parse(json['lastTrackerUpdateTime'] as String)
|
|
||||||
..customTrackers = (json['customTrackers'] as List<dynamic>).map((e) => e as String).toList();
|
|
||||||
|
|
||||||
Map<String, dynamic> _$ExtraConfigBtToJson(ExtraConfigBt instance) => <String, dynamic>{
|
|
||||||
'trackerSubscribeUrls': instance.trackerSubscribeUrls,
|
|
||||||
'subscribeTrackers': instance.subscribeTrackers,
|
|
||||||
'autoUpdateTrackers': instance.autoUpdateTrackers,
|
|
||||||
'lastTrackerUpdateTime': ?instance.lastTrackerUpdateTime?.toIso8601String(),
|
|
||||||
'customTrackers': instance.customTrackers,
|
|
||||||
};
|
|
||||||
|
|
||||||
GithubMirror _$GithubMirrorFromJson(Map<String, dynamic> json) => GithubMirror(
|
GithubMirror _$GithubMirrorFromJson(Map<String, dynamic> json) => GithubMirror(
|
||||||
type: $enumDecode(_$GithubMirrorTypeEnumMap, json['type']),
|
type: $enumDecode(_$GithubMirrorTypeEnumMap, json['type']),
|
||||||
url: json['url'] as String,
|
url: json['url'] as String,
|
||||||
@@ -282,14 +295,14 @@ Map<String, dynamic> _$GithubMirrorToJson(GithubMirror instance) => <String, dyn
|
|||||||
|
|
||||||
const _$GithubMirrorTypeEnumMap = {GithubMirrorType.jsdelivr: 'jsdelivr', GithubMirrorType.ghProxy: 'ghProxy'};
|
const _$GithubMirrorTypeEnumMap = {GithubMirrorType.jsdelivr: 'jsdelivr', GithubMirrorType.ghProxy: 'ghProxy'};
|
||||||
|
|
||||||
ExtraConfigGithubMirror _$ExtraConfigGithubMirrorFromJson(Map<String, dynamic> json) => ExtraConfigGithubMirror(
|
GithubMirrorConfig _$GithubMirrorConfigFromJson(Map<String, dynamic> json) => GithubMirrorConfig(
|
||||||
enabled: json['enabled'] as bool? ?? true,
|
enabled: json['enabled'] as bool? ?? true,
|
||||||
mirrors:
|
mirrors:
|
||||||
(json['mirrors'] as List<dynamic>?)?.map((e) => GithubMirror.fromJson(e as Map<String, dynamic>)).toList() ??
|
(json['mirrors'] as List<dynamic>?)?.map((e) => GithubMirror.fromJson(e as Map<String, dynamic>)).toList() ??
|
||||||
const [],
|
const [],
|
||||||
);
|
);
|
||||||
|
|
||||||
Map<String, dynamic> _$ExtraConfigGithubMirrorToJson(ExtraConfigGithubMirror instance) => <String, dynamic>{
|
Map<String, dynamic> _$GithubMirrorConfigToJson(GithubMirrorConfig instance) => <String, dynamic>{
|
||||||
'enabled': instance.enabled,
|
'enabled': instance.enabled,
|
||||||
'mirrors': instance.mirrors.map((e) => e.toJson()).toList(),
|
'mirrors': instance.mirrors.map((e) => e.toJson()).toList(),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -317,12 +317,7 @@ class AppPlatformController extends AsyncNotifier<AppPlatformState> with WindowL
|
|||||||
}
|
}
|
||||||
|
|
||||||
Future<void> startUpdate(VersionInfo versionInfo, UpdateProgressCallback onProgress) async {
|
Future<void> startUpdate(VersionInfo versionInfo, UpdateProgressCallback onProgress) async {
|
||||||
final runtime = ref.read(appRuntimeControllerProvider).value;
|
await updateApp(versionInfo, service: ref.read(gopeedServiceProvider), onProgress: onProgress);
|
||||||
await updateApp(
|
|
||||||
versionInfo,
|
|
||||||
githubMirror: runtime?.downloaderConfig.extra.githubMirror ?? ExtraConfigGithubMirror(enabled: false),
|
|
||||||
onProgress: onProgress,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
AppPlatformState _snapshot() => AppPlatformState(
|
AppPlatformState _snapshot() => AppPlatformState(
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
import 'dart:async';
|
import 'dart:async';
|
||||||
import 'dart:convert';
|
|
||||||
import 'package:flutter/foundation.dart';
|
import 'package:flutter/foundation.dart';
|
||||||
import 'package:flutter_riverpod/flutter_riverpod.dart';
|
import 'package:flutter_riverpod/flutter_riverpod.dart';
|
||||||
import 'package:path/path.dart' as path;
|
import 'package:path/path.dart' as path;
|
||||||
@@ -106,14 +105,9 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
|
|||||||
),
|
),
|
||||||
);
|
);
|
||||||
unawaited(LocationKeepAliveCoordinator.instance.reconcile(enabled: config.extra.backgroundLocationKeepAlive));
|
unawaited(LocationKeepAliveCoordinator.instance.reconcile(enabled: config.extra.backgroundLocationKeepAlive));
|
||||||
unawaited(
|
unawaited(ContinuedProcessing.setEnabled(config.extra.backgroundContinuedProcessing));
|
||||||
ContinuedProcessing.setEnabled(
|
|
||||||
config.extra.backgroundContinuedProcessing,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Future<AppRuntimeState> _init() async {
|
Future<AppRuntimeState> _init() async {
|
||||||
await AppInitializer.ensureStorageInitialized();
|
await AppInitializer.ensureStorageInitialized();
|
||||||
try {
|
try {
|
||||||
@@ -175,7 +169,6 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
|
|||||||
..address = config.api.address
|
..address = config.api.address
|
||||||
..apiToken = config.api.token;
|
..apiToken = config.api.token;
|
||||||
}
|
}
|
||||||
unawaited(_initTrackerUpdate(config));
|
|
||||||
unawaited(_initAndroidForegroundService(appLocalizationsFor(config.extra.locale)));
|
unawaited(_initAndroidForegroundService(appLocalizationsFor(config.extra.locale)));
|
||||||
await _initAnalytics(config);
|
await _initAnalytics(config);
|
||||||
|
|
||||||
@@ -190,9 +183,7 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
|
|||||||
// Make sure the native Continued Processing manager has the
|
// Make sure the native Continued Processing manager has the
|
||||||
// persisted setting before app initialization completes. This
|
// persisted setting before app initialization completes. This
|
||||||
// avoids the first download racing ahead of setEnabled(true).
|
// avoids the first download racing ahead of setEnabled(true).
|
||||||
await ContinuedProcessing.setEnabled(
|
await ContinuedProcessing.setEnabled(config.extra.backgroundContinuedProcessing);
|
||||||
config.extra.backgroundContinuedProcessing,
|
|
||||||
);
|
|
||||||
|
|
||||||
LocationKeepAliveCoordinator.instance.start(
|
LocationKeepAliveCoordinator.instance.start(
|
||||||
() =>
|
() =>
|
||||||
@@ -271,9 +262,6 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
|
|||||||
if (extra.themeColor.isEmpty) {
|
if (extra.themeColor.isEmpty) {
|
||||||
extra.themeColor = 'green';
|
extra.themeColor = 'green';
|
||||||
}
|
}
|
||||||
if (extra.bt.trackerSubscribeUrls.isEmpty) {
|
|
||||||
extra.bt.trackerSubscribeUrls.addAll(allTrackerSubscribeUrls);
|
|
||||||
}
|
|
||||||
if (config.proxy.scheme.isEmpty) {
|
if (config.proxy.scheme.isEmpty) {
|
||||||
config.proxy.scheme = 'http';
|
config.proxy.scheme = 'http';
|
||||||
}
|
}
|
||||||
@@ -308,26 +296,6 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
|
|||||||
),
|
),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
if (extra.githubMirror.mirrors.isEmpty) {
|
|
||||||
extra.githubMirror.mirrors = [
|
|
||||||
GithubMirror(type: GithubMirrorType.jsdelivr, url: 'https://fastly.jsdelivr.net/gh', isBuiltIn: true),
|
|
||||||
GithubMirror(type: GithubMirrorType.ghProxy, url: 'https://fastgit.cc', isBuiltIn: true),
|
|
||||||
];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> _initTrackerUpdate(DownloaderConfig config) async {
|
|
||||||
final btExtra = config.extra.bt;
|
|
||||||
final lastUpdateTime = btExtra.lastTrackerUpdateTime;
|
|
||||||
if (!btExtra.autoUpdateTrackers ||
|
|
||||||
(lastUpdateTime != null && !lastUpdateTime.isBefore(DateTime.now().subtract(const Duration(days: 1))))) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
await updateTrackers(config);
|
|
||||||
} catch (error, stackTrace) {
|
|
||||||
logger.w('tracker update failed', error, stackTrace);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<void> _initAnalytics(DownloaderConfig config) async {
|
Future<void> _initAnalytics(DownloaderConfig config) async {
|
||||||
@@ -344,32 +312,12 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<void> updateTrackers(DownloaderConfig config) async {
|
Future<void> updateTrackers() async {
|
||||||
final btExtra = config.extra.bt;
|
final service = ref.read(gopeedServiceProvider);
|
||||||
final trackers = <String>[];
|
try {
|
||||||
for (final url in btExtra.trackerSubscribeUrls) {
|
await service.triggerProtocolJob('bt', 'update-trackers');
|
||||||
final resp = await api.proxyRequest(url);
|
} finally {
|
||||||
final data = resp.data;
|
replaceDownloaderConfig(await service.getConfig());
|
||||||
if (resp.statusCode == 200 && data != null && data.isNotEmpty) {
|
|
||||||
trackers.addAll(const LineSplitter().convert(data).where((line) => line.trim().isNotEmpty));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
btExtra.subscribeTrackers = trackers;
|
|
||||||
btExtra.lastTrackerUpdateTime = DateTime.now();
|
|
||||||
config.protocolConfig.bt.trackers = {...btExtra.subscribeTrackers, ...btExtra.customTrackers}.toList();
|
|
||||||
await ref.read(gopeedServiceProvider).putConfig(config);
|
|
||||||
|
|
||||||
final current = state.value;
|
|
||||||
if (current != null) {
|
|
||||||
state = AsyncValue.data(
|
|
||||||
AppRuntimeState(
|
|
||||||
startConfig: current.startConfig,
|
|
||||||
apiServerState: current.apiServerState,
|
|
||||||
downloaderConfig: config,
|
|
||||||
localBackendStarted: current.localBackendStarted,
|
|
||||||
startupError: current.startupError,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -509,17 +457,3 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
|
|||||||
return operation;
|
return operation;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const allTrackerSubscribeUrls = [
|
|
||||||
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt',
|
|
||||||
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt',
|
|
||||||
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_https.txt',
|
|
||||||
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt',
|
|
||||||
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_udp.txt',
|
|
||||||
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ws.txt',
|
|
||||||
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best.txt',
|
|
||||||
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best_ip.txt',
|
|
||||||
'https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/all.txt',
|
|
||||||
'https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best.txt',
|
|
||||||
'https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/http.txt',
|
|
||||||
];
|
|
||||||
@@ -82,6 +82,14 @@ RpcCodecRegistry createAppCapabilityCodecs() {
|
|||||||
|
|
||||||
void _bindGopeed(CapabilityRegistry registry) {
|
void _bindGopeed(CapabilityRegistry registry) {
|
||||||
registry
|
registry
|
||||||
|
..bind(GopeedMethods.triggerProtocolJob, (p) async {
|
||||||
|
await api.triggerProtocolJob(p['protocol'] as String, p['name'] as String);
|
||||||
|
return const RpcUnit();
|
||||||
|
})
|
||||||
|
..bind(
|
||||||
|
GopeedMethods.getMirrorCandidates,
|
||||||
|
(p) => api.getMirrorCandidates(p['provider'] as String, p['url'] as String),
|
||||||
|
)
|
||||||
..bind(GopeedMethods.resolve, api.resolve)
|
..bind(GopeedMethods.resolve, api.resolve)
|
||||||
..bind(GopeedMethods.createTask, api.createTask)
|
..bind(GopeedMethods.createTask, api.createTask)
|
||||||
..bind(GopeedMethods.createTaskBatch, api.createTaskBatch)
|
..bind(GopeedMethods.createTaskBatch, api.createTaskBatch)
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ import '../../api/model/update_extension_settings.dart';
|
|||||||
import 'capability_rpc.dart';
|
import 'capability_rpc.dart';
|
||||||
|
|
||||||
abstract final class GopeedMethods {
|
abstract final class GopeedMethods {
|
||||||
|
static const triggerProtocolJob = RpcMethod<Map<String, dynamic>, RpcUnit>('gopeed.protocol.job.trigger');
|
||||||
|
static const getMirrorCandidates = RpcMethod<Map<String, dynamic>, List<String>>('gopeed.mirrors.candidates');
|
||||||
static const resolve = RpcMethod<ResolveTask, ResolveResult>('gopeed.resolve');
|
static const resolve = RpcMethod<ResolveTask, ResolveResult>('gopeed.resolve');
|
||||||
static const createTask = RpcMethod<CreateTask, String>('gopeed.task.create');
|
static const createTask = RpcMethod<CreateTask, String>('gopeed.task.create');
|
||||||
static const createTaskBatch = RpcMethod<CreateTaskBatch, List<String>>('gopeed.task.createBatch');
|
static const createTaskBatch = RpcMethod<CreateTaskBatch, List<String>>('gopeed.task.createBatch');
|
||||||
@@ -43,6 +45,10 @@ class GopeedService {
|
|||||||
|
|
||||||
final CapabilityInvoker _invoker;
|
final CapabilityInvoker _invoker;
|
||||||
|
|
||||||
|
Future<void> triggerProtocolJob(String protocol, String name) async =>
|
||||||
|
_invoker.invoke(GopeedMethods.triggerProtocolJob, {'protocol': protocol, 'name': name});
|
||||||
|
Future<List<String>> getMirrorCandidates(String provider, String url) =>
|
||||||
|
_invoker.invoke(GopeedMethods.getMirrorCandidates, {'provider': provider, 'url': url});
|
||||||
Future<ResolveResult> resolve(ResolveTask request) => _invoker.invoke(GopeedMethods.resolve, request);
|
Future<ResolveResult> resolve(ResolveTask request) => _invoker.invoke(GopeedMethods.resolve, request);
|
||||||
|
|
||||||
Future<String> createTask(CreateTask request) => _invoker.invoke(GopeedMethods.createTask, request);
|
Future<String> createTask(CreateTask request) => _invoker.invoke(GopeedMethods.createTask, request);
|
||||||
|
|||||||
@@ -0,0 +1,147 @@
|
|||||||
|
import 'package:dio/dio.dart';
|
||||||
|
import 'package:pub_semver/pub_semver.dart';
|
||||||
|
|
||||||
|
import '../../api/gopeed_site_api.dart';
|
||||||
|
|
||||||
|
class VersionInfo {
|
||||||
|
const VersionInfo({
|
||||||
|
required this.version,
|
||||||
|
required this.changeLog,
|
||||||
|
required this.releaseUrl,
|
||||||
|
this.assetSha256 = const {},
|
||||||
|
});
|
||||||
|
|
||||||
|
final String version;
|
||||||
|
final String changeLog;
|
||||||
|
final String releaseUrl;
|
||||||
|
// Populated only from official release metadata, never from a download mirror.
|
||||||
|
final Map<String, String> assetSha256;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Query the official GitHub API directly, then the independent website API
|
||||||
|
/// only if the GitHub request or its response validation fails.
|
||||||
|
class AppUpdateClient {
|
||||||
|
AppUpdateClient({Dio? client, GopeedSiteApi? site})
|
||||||
|
: _client =
|
||||||
|
client ??
|
||||||
|
Dio(BaseOptions(connectTimeout: const Duration(seconds: 10), receiveTimeout: const Duration(seconds: 30))),
|
||||||
|
_site = site ?? GopeedSiteApi.instance;
|
||||||
|
|
||||||
|
static const releasesUrl = 'https://api.github.com/repos/GopeedLab/gopeed/releases?per_page=10';
|
||||||
|
final Dio _client;
|
||||||
|
final GopeedSiteApi _site;
|
||||||
|
|
||||||
|
Future<List<_Release>> _fetchGithubReleases() async {
|
||||||
|
final cancel = CancelToken();
|
||||||
|
try {
|
||||||
|
final response = await _client
|
||||||
|
.get<dynamic>(releasesUrl, cancelToken: cancel, options: Options(followRedirects: false))
|
||||||
|
.timeout(const Duration(seconds: 30));
|
||||||
|
return _decodeReleases(response.data);
|
||||||
|
} finally {
|
||||||
|
cancel.cancel('GitHub release request completed');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<VersionInfo?> check(String currentVersion) async {
|
||||||
|
final current = _parseVersion(currentVersion);
|
||||||
|
if (current == null) throw FormatException('Invalid current application version', currentVersion);
|
||||||
|
List<_Release> releases;
|
||||||
|
try {
|
||||||
|
releases = await _fetchGithubReleases();
|
||||||
|
} catch (_) {
|
||||||
|
releases = _decodeReleases(await _site.getReleases());
|
||||||
|
}
|
||||||
|
_Release? selected;
|
||||||
|
for (final release in releases) {
|
||||||
|
final version = release.version;
|
||||||
|
if (release.draft || version.compareTo(current) <= 0) continue;
|
||||||
|
final preview = release.prerelease || version.isPreRelease;
|
||||||
|
if (!current.isPreRelease && preview) continue;
|
||||||
|
if (current.isPreRelease &&
|
||||||
|
preview &&
|
||||||
|
(version.major != current.major || version.minor != current.minor || version.patch != current.patch)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (selected == null || version.compareTo(selected.version) > 0) selected = release;
|
||||||
|
}
|
||||||
|
if (selected == null) return null;
|
||||||
|
return VersionInfo(
|
||||||
|
version: selected.tag.replaceFirst(RegExp(r'^v'), ''),
|
||||||
|
changeLog: selected.body,
|
||||||
|
assetSha256: Map.unmodifiable(selected.assetSha256),
|
||||||
|
releaseUrl: selected.url.isEmpty
|
||||||
|
? 'https://github.com/GopeedLab/gopeed/releases/tag/${selected.tag}'
|
||||||
|
: selected.url,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build metadata does not affect SemVer precedence.
|
||||||
|
Version? _parseVersion(String raw) {
|
||||||
|
try {
|
||||||
|
final version = Version.parse(raw.replaceFirst(RegExp(r'^v'), ''));
|
||||||
|
return Version(version.major, version.minor, version.patch, pre: version.preRelease.join('.'));
|
||||||
|
} on FormatException {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
List<_Release> _decodeReleases(dynamic data) {
|
||||||
|
if (data is! List) throw const FormatException('Invalid release list');
|
||||||
|
final releases = <_Release>[];
|
||||||
|
for (final entry in data) {
|
||||||
|
if (entry is! Map || entry['tag_name'] is! String) continue;
|
||||||
|
final tag = entry['tag_name'] as String;
|
||||||
|
final version = _parseVersion(tag);
|
||||||
|
if (version == null) continue;
|
||||||
|
releases.add(
|
||||||
|
_Release(
|
||||||
|
tag: tag,
|
||||||
|
version: version,
|
||||||
|
draft: entry['draft'] == true,
|
||||||
|
prerelease: entry['prerelease'] == true,
|
||||||
|
body: entry['body'] as String? ?? '',
|
||||||
|
url: entry['html_url'] as String? ?? '',
|
||||||
|
assetSha256: _decodeAssetSha256(entry['assets']),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (data.isNotEmpty && releases.isEmpty) throw const FormatException('Response contains no valid releases');
|
||||||
|
return releases;
|
||||||
|
}
|
||||||
|
|
||||||
|
Map<String, String> _decodeAssetSha256(dynamic assets) {
|
||||||
|
if (assets == null) return {};
|
||||||
|
if (assets is! List) throw const FormatException('Invalid release assets');
|
||||||
|
final result = <String, String>{};
|
||||||
|
for (final asset in assets) {
|
||||||
|
if (asset is! Map || asset['name'] is! String || asset['digest'] is! String) continue;
|
||||||
|
final digest = asset['digest'] as String;
|
||||||
|
final match = RegExp(r'^sha256:([a-fA-F0-9]{64})$').firstMatch(digest);
|
||||||
|
if (match == null || match.end != digest.length) continue;
|
||||||
|
final name = asset['name'] as String;
|
||||||
|
if (result.containsKey(name)) throw const FormatException('Duplicate release asset');
|
||||||
|
result[name] = match.group(1)!.toLowerCase();
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
class _Release {
|
||||||
|
const _Release({
|
||||||
|
required this.tag,
|
||||||
|
required this.version,
|
||||||
|
required this.draft,
|
||||||
|
required this.prerelease,
|
||||||
|
required this.body,
|
||||||
|
required this.url,
|
||||||
|
required this.assetSha256,
|
||||||
|
});
|
||||||
|
final String tag;
|
||||||
|
final Version version;
|
||||||
|
final bool draft;
|
||||||
|
final bool prerelease;
|
||||||
|
final String body;
|
||||||
|
final String url;
|
||||||
|
final Map<String, String> assetSha256;
|
||||||
|
}
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
import 'dart:io';
|
||||||
|
|
||||||
|
import 'package:crypto/crypto.dart';
|
||||||
|
import 'package:dio/dio.dart';
|
||||||
|
|
||||||
|
class DownloadChecksumException implements Exception {
|
||||||
|
const DownloadChecksumException();
|
||||||
|
@override
|
||||||
|
String toString() => 'Update package checksum verification failed';
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Probes backend-provided URLs concurrently, then downloads the fastest usable
|
||||||
|
/// candidate. Other candidates remain available if HEAD is unsupported or the
|
||||||
|
/// actual download fails. This layer has no GitHub URL or mirror-config rules.
|
||||||
|
Future<void> downloadFromCandidates(
|
||||||
|
List<String> candidates,
|
||||||
|
String savePath, {
|
||||||
|
required ProgressCallback onReceiveProgress,
|
||||||
|
required String expectedSha256,
|
||||||
|
Dio? client,
|
||||||
|
Duration probeTimeout = const Duration(seconds: 3),
|
||||||
|
}) async {
|
||||||
|
// Reject missing or malformed trusted metadata before any network request.
|
||||||
|
if (expectedSha256.length != 64 || !RegExp(r'^[a-fA-F0-9]{64}$').hasMatch(expectedSha256)) {
|
||||||
|
throw const DownloadChecksumException();
|
||||||
|
}
|
||||||
|
final urls = candidates.toSet().toList(growable: false);
|
||||||
|
if (urls.isEmpty) throw ArgumentError.value(candidates, 'candidates', 'No download candidates');
|
||||||
|
final dio = client ?? Dio(BaseOptions(connectTimeout: const Duration(seconds: 10)));
|
||||||
|
try {
|
||||||
|
final probes = await Future.wait(
|
||||||
|
urls.map((url) async {
|
||||||
|
final watch = Stopwatch()..start();
|
||||||
|
final cancel = CancelToken();
|
||||||
|
try {
|
||||||
|
final response = await dio
|
||||||
|
.head<void>(
|
||||||
|
url,
|
||||||
|
cancelToken: cancel,
|
||||||
|
options: Options(receiveTimeout: probeTimeout, validateStatus: (status) => status == 200),
|
||||||
|
)
|
||||||
|
.timeout(probeTimeout);
|
||||||
|
if (_isPackageResponse(response.headers)) return (url: url, latency: watch.elapsed);
|
||||||
|
} catch (_) {
|
||||||
|
// HEAD failure does not prove that a normal GET will fail.
|
||||||
|
} finally {
|
||||||
|
cancel.cancel();
|
||||||
|
}
|
||||||
|
return (url: url, latency: null);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
final available = probes.where((probe) => probe.latency != null).toList()
|
||||||
|
..sort((a, b) => a.latency!.compareTo(b.latency!));
|
||||||
|
final ordered = [
|
||||||
|
...available.map((probe) => probe.url),
|
||||||
|
...probes.where((p) => p.latency == null).map((p) => p.url),
|
||||||
|
];
|
||||||
|
Object? lastError;
|
||||||
|
StackTrace? lastStack;
|
||||||
|
for (final url in ordered) {
|
||||||
|
try {
|
||||||
|
onReceiveProgress(0, 0);
|
||||||
|
final response = await dio.download(
|
||||||
|
url,
|
||||||
|
savePath,
|
||||||
|
onReceiveProgress: onReceiveProgress,
|
||||||
|
deleteOnError: true,
|
||||||
|
options: Options(receiveTimeout: const Duration(seconds: 30), validateStatus: (status) => status == 200),
|
||||||
|
);
|
||||||
|
if (!_isPackageResponse(response.headers) || await File(savePath).length() == 0) {
|
||||||
|
throw StateError('Invalid download response from $url');
|
||||||
|
}
|
||||||
|
final actual = await sha256.bind(File(savePath).openRead()).first;
|
||||||
|
if (actual.toString() != expectedSha256.toLowerCase()) throw const DownloadChecksumException();
|
||||||
|
return;
|
||||||
|
} catch (error, stack) {
|
||||||
|
lastError = error;
|
||||||
|
lastStack = stack;
|
||||||
|
final file = File(savePath);
|
||||||
|
if (await file.exists()) await file.delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Error.throwWithStackTrace(lastError!, lastStack!);
|
||||||
|
} finally {
|
||||||
|
if (client == null) dio.close(force: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool _isPackageResponse(Headers headers) {
|
||||||
|
final contentType = headers.value(Headers.contentTypeHeader)?.split(';').first.trim().toLowerCase();
|
||||||
|
return contentType != 'text/html' && contentType != 'application/xhtml+xml' && contentType != 'application/json';
|
||||||
|
}
|
||||||
Loaded 50 of 98 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user