refactor: remove proxy API (#1596)

This commit is contained in:
Levi authored and GitHub committed 2026-10-21 22:48:17 +08:00
1 parent 47cafb7575
commit 4214765d39
98 files changed
+5883 -1150

No files matched your search

+1 -1
View File
@@ -66,6 +66,7 @@ require (
go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect
go.uber.org/mock v0.5.1 // indirect go.uber.org/mock v0.5.1 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/mod v0.33.0
golang.org/x/oauth2 v0.35.0 // indirect golang.org/x/oauth2 v0.35.0 // indirect
golang.org/x/sync v0.20.0 // indirect golang.org/x/sync v0.20.0 // indirect
) )
@@ -156,7 +157,6 @@ require (
go.opentelemetry.io/otel/metric v1.39.0 // indirect go.opentelemetry.io/otel/metric v1.39.0 // indirect
go.opentelemetry.io/otel/trace v1.39.0 // indirect go.opentelemetry.io/otel/trace v1.39.0 // indirect
golang.org/x/crypto v0.48.0 // indirect golang.org/x/crypto v0.48.0 // indirect
golang.org/x/mod v0.33.0 // indirect
golang.org/x/net v0.50.0 golang.org/x/net v0.50.0
golang.org/x/sys v0.44.0 // indirect golang.org/x/sys v0.44.0 // indirect
golang.org/x/text v0.34.0 golang.org/x/text v0.34.0
+72
View File
@@ -0,0 +1,72 @@
package fetcher
import (
"context"
"encoding/json"
"errors"
"time"
)
type TriggerSource string
const (
TriggerScheduled TriggerSource = "scheduled"
TriggerManual TriggerSource = "manual"
)
var ErrJobSkipped = errors.New("protocol job skipped")
var ErrConfigChanged = errors.New("protocol configuration changed during execution")
// ConfigState embeds the common backend-owned state section in a protocol config.
// Public config PUTs always retain this section from the latest stored config.
type ConfigState[T any] struct {
State T `json:"state"`
}
// ProtocolHost is scoped to one manager. UpdateState runs against the latest
// config under Downloader's lock; callbacks must not perform network I/O.
type ProtocolHost struct {
Read func(any) error
UpdateState func(func(config, state json.RawMessage) (any, error)) error
Fetch func(context.Context, string, int64, func([]byte) error) ([]byte, error)
}
func ReadConfig[T any](host ProtocolHost) (*T, error) {
var cfg T
if err := host.Read(&cfg); err != nil {
return nil, err
}
return &cfg, nil
}
// UpdateState gives the callback a detached config snapshot and mutable state.
// Only the state is committed; edits to the config snapshot are discarded.
func UpdateState[C, S any](host ProtocolHost, update func(*C, *S) error) error {
return host.UpdateState(func(rawConfig, rawState json.RawMessage) (any, error) {
var cfg C
var state S
if err := json.Unmarshal(rawConfig, &cfg); err != nil {
return nil, err
}
if err := json.Unmarshal(rawState, &state); err != nil {
return nil, err
}
if err := update(&cfg, &state); err != nil {
return nil, err
}
return &state, nil
})
}
type CronJob struct {
Name string
Every time.Duration
Timeout time.Duration
Run func(context.Context, TriggerSource) error
}
type CronProvider interface{ CronJobs(ProtocolHost) []CronJob }
// ConfigNormalizer is optional. It owns protocol validation and derived fields.
type ConfigNormalizer interface {
NormalizeConfig(json.RawMessage) (any, error)
}
+73
View File
@@ -0,0 +1,73 @@
# GitHub mirror execution
This package owns GitHub URL rewriting and execution strategies. It depends only
on the standard library and does not depend on Downloader, HTTP clients, Git, or
storage. Downloader persists `Config` at its top-level `githubMirror` field;
`pkg/base` preserves the existing configuration names through aliases.
## Race: resource reads
```go
body, err := githubmirror.Race(ctx, cfg.GithubMirror, rawURL,
githubmirror.Options[[]byte]{
Kind: githubmirror.Resource,
Timeout: 30 * time.Second,
},
func(ctx context.Context, candidate string) ([]byte, error) {
return fetchAndValidate(ctx, client, candidate)
},
)
```
The callback owns HTTP requests, size limits, and content validation. The first
completed valid result wins. Other operations are canceled and drained. There is
no HEAD-probe assumption or client-specific HTTP behavior inside the executor.
## Fallback: repository operations
```go
repo, err := githubmirror.Fallback(ctx, cfg.GithubMirror, officialURL,
githubmirror.Options[*git.Repository]{
Kind: githubmirror.Repository,
},
func(ctx context.Context, candidate string) (*git.Repository, error) {
// Use a clean temporary checkout for each attempt.
return clone(ctx, candidate)
},
)
```
The official URL is attempted first; mirrors follow their configured order.
jsDelivr is excluded for repositories. Stop on the first success. Perform
persistent installation or other business state changes after success, rather
than inside a potentially repeated attempt.
## Results with resources
For temporary files, set `Options[*Download].Discard` to remove the file. Return
the partial `*Download` alongside errors, so cleanup also covers failed reads or
validation. The executor discards every failure and every race loser exactly
once. The caller owns and must track/release the winning result.
- `Timeout == 0` uses 30 seconds per resource attempt and no overall deadline
for repository attempts; a negative timeout is invalid. Extension Git HTTP
transport limits connections/TLS handshakes to 10 seconds and resets a 60-second
read deadline as data arrives. Slow continuous transfers can finish, while
stalled sources fail and allow ordered fallback. Parent cancellation still applies.
- Attempts must honor the supplied context; draining waits for them to return.
- Attempt contexts are canceled after return, so callbacks must finish reading
HTTP bodies before returning. Returning a still-streaming response is unsupported.
- Discard callbacks must accept zero values, must not panic, and run serially.
- Parent cancellation wins over a result selected while the parent is canceled;
the selected result is then discarded as well.
- Errors preserve their causes through wrapping/joining for `errors.Is`.
- `github.com` and `raw.githubusercontent.com` expand into eligible mirrors.
jsDelivr is only eligible for raw source resources. `api.github.com` is never
mirrored and is rejected by the public mirror candidates API.
- Client application update checks request the official GitHub API directly,
falling back to the independent Gopeed website API on request or validation
failure. Only package downloads obtain URLs from
`POST /api/v1/mirrors/github/candidates`. Candidate generation never performs
HTTP requests.
- Disabled mirrors and other URLs execute only the original URL.
- Config is a caller-owned immutable snapshot for the duration of a call.
+90
View File
@@ -0,0 +1,90 @@
// Package githubmirror rewrites GitHub URLs and executes mirrored operations.
package githubmirror
import (
"fmt"
"net/url"
"strings"
)
type Mirror struct {
Type string `json:"type"`
URL string `json:"url"`
IsBuiltIn bool `json:"isBuiltIn"`
IsDeleted bool `json:"isDeleted"`
}
type Config struct {
Enabled bool `json:"enabled"`
Mirrors []Mirror `json:"mirrors"`
}
func DefaultConfig() *Config {
return &Config{Enabled: true, Mirrors: []Mirror{
{Type: "jsdelivr", URL: "https://fastly.jsdelivr.net/gh", IsBuiltIn: true},
{Type: "ghProxy", URL: "https://fastgit.cc", IsBuiltIn: true},
{Type: "ghProxy", URL: "https://gh-proxy.com", IsBuiltIn: true},
{Type: "ghProxy", URL: "https://ghfast.top", IsBuiltIn: true},
}}
}
func (c *Config) Validate() error {
if len(c.Mirrors) > 32 {
return fmt.Errorf("too many GitHub mirrors")
}
for _, mirror := range c.Mirrors {
u, err := url.Parse(mirror.URL)
if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
return fmt.Errorf("invalid GitHub mirror URL")
}
if mirror.Type != "jsdelivr" && mirror.Type != "ghProxy" {
return fmt.Errorf("invalid GitHub mirror type: %s", mirror.Type)
}
}
return nil
}
// Kind describes what a candidate endpoint must support.
type Kind uint8
const (
Resource Kind = iota
Repository
)
// Candidates preserves the official URL first. jsDelivr is only eligible for
// raw source resources; Git operations use mirrors supporting Git Smart HTTP.
// GitHub API URLs are never mirrored.
func Candidates(c *Config, raw string, kind Kind) []string {
result := []string{raw}
u, err := url.Parse(raw)
if err != nil || c == nil || !c.Enabled || u.User != nil || (u.Scheme != "https" && u.Scheme != "http") {
return result
}
if u.Host != "github.com" && u.Host != "raw.githubusercontent.com" {
return result
}
seen := map[string]bool{raw: true}
for _, mirror := range c.Mirrors {
if mirror.IsDeleted {
continue
}
var candidate string
switch mirror.Type {
case "ghProxy":
candidate = strings.TrimRight(mirror.URL, "/") + "/" + raw
case "jsdelivr":
if kind == Repository || u.Host != "raw.githubusercontent.com" {
continue
}
parts := strings.SplitN(strings.TrimPrefix(u.EscapedPath(), "/"), "/", 4)
if len(parts) != 4 {
continue
}
candidate = strings.TrimRight(mirror.URL, "/") + "/" + parts[0] + "/" + parts[1] + "@" + parts[2] + "/" + parts[3]
}
if candidate != "" && !seen[candidate] {
result = append(result, candidate)
seen[candidate] = true
}
}
return result
}
+38
View File
@@ -0,0 +1,38 @@
package githubmirror
import (
"reflect"
"testing"
)
func TestMirrorCandidates(t *testing.T) {
cfg := &Config{Enabled: true, Mirrors: []Mirror{
{Type: "jsdelivr", URL: "https://cdn.example/gh/"},
{Type: "ghProxy", URL: "https://mirror.example/"},
{Type: "ghProxy", URL: "https://deleted.example", IsDeleted: true},
}}
source := "https://raw.githubusercontent.com/owner/repo/main/a.txt"
want := []string{source, "https://cdn.example/gh/owner/repo@main/a.txt", "https://mirror.example/" + source}
if got := Candidates(cfg, source, Resource); !reflect.DeepEqual(got, want) {
t.Fatalf("got %v", got)
}
repo := "https://github.com/owner/repo.git"
if got := Candidates(cfg, repo, Repository); !reflect.DeepEqual(got, []string{repo, "https://mirror.example/" + repo}) {
t.Fatalf("git candidates %v", got)
}
for _, raw := range []string{"https://github.com.evil.example/a", "https://example.com/a", "https://user:[email protected]/a"} {
if got := Candidates(cfg, raw, Resource); !reflect.DeepEqual(got, []string{raw}) {
t.Fatalf("unexpected mirror: %v", got)
}
}
apiURL := "https://api.github.com/repos/GopeedLab/gopeed/releases?per_page=10"
for _, kind := range []Kind{Resource, Repository} {
if got := Candidates(cfg, apiURL, kind); !reflect.DeepEqual(got, []string{apiURL}) {
t.Fatalf("GitHub API must never use mirrors: %v", got)
}
}
cfg.Enabled = false
if got := Candidates(cfg, source, Resource); !reflect.DeepEqual(got, []string{source}) {
t.Fatalf("disabled mirror: %v", got)
}
}
+160
View File
@@ -0,0 +1,160 @@
package githubmirror
import (
"context"
"errors"
"fmt"
"time"
)
// Options configures execution, not URL persistence. Discard releases results
// from failed attempts and successful race losers, including partial results
// returned together with an error. The caller owns the winning result.
type Options[T any] struct {
Kind Kind
// Timeout limits each attempt. Zero uses 30 seconds for resources and no
// overall deadline for repositories, whose transport owns idle timeouts.
Timeout time.Duration
Discard func(T)
}
// Attempt must honor ctx and return a completed, validated result. Its context
// is canceled after it returns, so live response streams must not be returned.
// Return any partially acquired resources along with errors so Discard can
// release them. Failed attempts must not mutate persistent business state.
type Attempt[T any] func(context.Context, string) (T, error)
const defaultTimeout = 30 * time.Second
func prepare[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) ([]string, time.Duration, error) {
if err := ctx.Err(); err != nil {
return nil, 0, err
}
if raw == "" {
return nil, 0, errors.New("empty GitHub source URL")
}
if options.Kind != Resource && options.Kind != Repository {
return nil, 0, errors.New("invalid GitHub resource kind")
}
if options.Timeout < 0 {
return nil, 0, errors.New("invalid GitHub attempt timeout")
}
if attempt == nil {
return nil, 0, errors.New("GitHub mirror attempt is required")
}
if config != nil {
if err := config.Validate(); err != nil {
return nil, 0, err
}
}
timeout := options.Timeout
if timeout == 0 && options.Kind == Resource {
timeout = defaultTimeout
}
return Candidates(config, raw, options.Kind), timeout, nil
}
func invoke[T any](ctx context.Context, timeout time.Duration, candidate string, attempt Attempt[T]) (value T, err error) {
var cancel context.CancelFunc
if timeout > 0 {
ctx, cancel = context.WithTimeout(ctx, timeout)
} else {
ctx, cancel = context.WithCancel(ctx)
}
defer cancel()
defer func() {
if recovered := recover(); recovered != nil {
err = fmt.Errorf("GitHub mirror attempt panicked: %v", recovered)
}
}()
value, err = attempt(ctx, candidate)
if err == nil {
err = ctx.Err()
}
return
}
func discard[T any](options Options[T], value T) {
if options.Discard != nil {
options.Discard(value)
}
}
// Race executes the official URL and eligible mirrors concurrently. The first
// completed successful result wins; losing attempts are canceled and drained
// before return so no temporary files or other resources outlive the call.
// HTTP status/content validation is the attempt's responsibility.
func Race[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) (T, error) {
var zero T
candidates, timeout, err := prepare(ctx, config, raw, options, attempt)
if err != nil {
return zero, err
}
raceCtx, cancel := context.WithCancel(ctx)
defer cancel()
type outcome struct {
index int
value T
err error
}
outcomes := make(chan outcome, len(candidates))
for index, candidate := range candidates {
go func(index int, candidate string) {
value, err := invoke(raceCtx, timeout, candidate, attempt)
outcomes <- outcome{index, value, err}
}(index, candidate)
}
var winner T
won := false
failures := make([]error, len(candidates))
for range candidates {
outcome := <-outcomes
if !won && outcome.err == nil && ctx.Err() == nil {
winner = outcome.value
won = true
cancel()
} else {
discard(options, outcome.value)
if outcome.err != nil {
failures[outcome.index] = fmt.Errorf("GitHub source %d: %w", outcome.index+1, outcome.err)
}
if ctx.Err() != nil {
cancel()
}
}
}
if err := ctx.Err(); err != nil {
if won {
discard(options, winner)
}
return zero, err
}
if won {
return winner, nil
}
return zero, fmt.Errorf("all GitHub sources failed: %w", errors.Join(failures...))
}
// Fallback executes the official URL first, then eligible mirrors in their
// configured order. It stops immediately on success and cleans every failure.
func Fallback[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) (T, error) {
var zero T
candidates, timeout, err := prepare(ctx, config, raw, options, attempt)
if err != nil {
return zero, err
}
failures := make([]error, 0, len(candidates))
for index, candidate := range candidates {
if err := ctx.Err(); err != nil {
return zero, err
}
value, err := invoke(ctx, timeout, candidate, attempt)
if err == nil && ctx.Err() == nil {
return value, nil
}
discard(options, value)
if parentErr := ctx.Err(); parentErr != nil {
return zero, parentErr
}
failures = append(failures, fmt.Errorf("GitHub source %d: %w", index+1, err))
}
return zero, fmt.Errorf("all GitHub sources failed: %w", errors.Join(failures...))
}
+214
View File
@@ -0,0 +1,214 @@
package githubmirror
import (
"context"
"errors"
"reflect"
"strings"
"sync"
"testing"
"time"
)
const testSource = "https://github.com/owner/repo.git"
func testConfig() *Config {
return &Config{Enabled: true, Mirrors: []Mirror{
{Type: "ghProxy", URL: "https://first.example"},
{Type: "ghProxy", URL: "https://second.example"},
}}
}
func TestRaceWinsAndDrainsOwnedResults(t *testing.T) {
config := testConfig()
candidates := Candidates(config, testSource, Resource)
var started sync.WaitGroup
started.Add(len(candidates))
discarded := []string{}
winner, err := Race(context.Background(), config, testSource, Options[string]{
Discard: func(value string) { discarded = append(discarded, value) },
}, func(ctx context.Context, address string) (string, error) {
started.Done()
started.Wait()
switch address {
case candidates[0]:
return address, errors.New("official offline")
case candidates[1]:
return address, nil
default:
<-ctx.Done()
return address, ctx.Err()
}
})
if err != nil || winner != candidates[1] {
t.Fatalf("winner=%s err=%v", winner, err)
}
if len(discarded) != 2 || !contains(discarded, candidates[0]) || !contains(discarded, candidates[2]) {
t.Fatalf("discarded %v", discarded)
}
}
func contains(values []string, value string) bool {
for _, item := range values {
if item == value {
return true
}
}
return false
}
func TestRaceCleansSimultaneousSuccessfulLosers(t *testing.T) {
config := testConfig()
var started sync.WaitGroup
started.Add(3)
cleaned := []string{}
winner, err := Race(context.Background(), config, testSource, Options[string]{Discard: func(value string) { cleaned = append(cleaned, value) }}, func(_ context.Context, address string) (string, error) {
started.Done()
started.Wait()
return address, nil
})
if err != nil || winner == "" || len(cleaned) != 2 || contains(cleaned, winner) {
t.Fatalf("winner=%s discarded=%v err=%v", winner, cleaned, err)
}
}
func TestRaceParentCancellationCleansWinnerToo(t *testing.T) {
config := testConfig()
config.Mirrors = config.Mirrors[:1]
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cleaned := []string{}
winner, err := Race(ctx, config, testSource, Options[string]{Discard: func(value string) { cleaned = append(cleaned, value) }}, func(ctx context.Context, address string) (string, error) {
if address == testSource {
return address, nil
}
<-ctx.Done()
cancel()
return address, ctx.Err()
})
if !errors.Is(err, context.Canceled) || winner != "" || len(cleaned) != 2 || !contains(cleaned, testSource) {
t.Fatalf("winner=%s cleanup=%v err=%v", winner, cleaned, err)
}
}
func TestRaceTimeoutAndErrorAggregation(t *testing.T) {
expected := errors.New("offline")
config := testConfig()
config.Mirrors = config.Mirrors[:1]
discarded := 0
_, err := Race(context.Background(), config, testSource, Options[string]{Timeout: 10 * time.Millisecond, Discard: func(string) { discarded++ }}, func(ctx context.Context, address string) (string, error) {
if address == testSource {
return address, expected
}
<-ctx.Done()
return address, ctx.Err()
})
if !errors.Is(err, expected) || !errors.Is(err, context.DeadlineExceeded) || discarded != 2 {
t.Fatalf("discarded=%d err=%v", discarded, err)
}
}
func TestFallbackPreservesOrderAndStopsOnSuccess(t *testing.T) {
config := testConfig()
candidates := Candidates(config, testSource, Repository)
tried := []string{}
cleaned := []string{}
winner, err := Fallback(context.Background(), config, testSource, Options[string]{Kind: Repository, Discard: func(value string) { cleaned = append(cleaned, value) }}, func(_ context.Context, address string) (string, error) {
tried = append(tried, address)
if address == candidates[1] {
return address, nil
}
return address, errors.New("offline")
})
if err != nil || winner != candidates[1] || !reflect.DeepEqual(tried, candidates[:2]) || !reflect.DeepEqual(cleaned, candidates[:1]) {
t.Fatalf("winner=%s tried=%v discarded=%v err=%v", winner, tried, cleaned, err)
}
tried = nil
_, err = Fallback(context.Background(), config, testSource, Options[string]{Kind: Repository}, func(_ context.Context, address string) (string, error) {
tried = append(tried, address)
return address, nil
})
if err != nil || !reflect.DeepEqual(tried, candidates[:1]) {
t.Fatalf("tried=%v err=%v", tried, err)
}
}
func TestFallbackTimeoutContinuesWithFreshContext(t *testing.T) {
discarded := []string{}
winner, err := Fallback(context.Background(), testConfig(), testSource, Options[string]{Kind: Repository, Timeout: 10 * time.Millisecond, Discard: func(value string) { discarded = append(discarded, value) }}, func(ctx context.Context, address string) (string, error) {
if address == testSource {
<-ctx.Done()
return address, ctx.Err()
}
if err := ctx.Err(); err != nil {
return address, err
}
return address, nil
})
if err != nil || !strings.HasPrefix(winner, "https://first.example/") || !reflect.DeepEqual(discarded, []string{testSource}) {
t.Fatalf("winner=%s cleanup=%v err=%v", winner, discarded, err)
}
}
func TestFallbackCancellationStopsBeforeNextMirror(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
calls := 0
cleaned := 0
_, err := Fallback(ctx, testConfig(), testSource, Options[string]{Kind: Repository, Discard: func(string) { cleaned++ }}, func(ctx context.Context, address string) (string, error) {
calls++
cancel()
return address, ctx.Err()
})
if !errors.Is(err, context.Canceled) || calls != 1 || cleaned != 1 {
t.Fatalf("calls=%d discarded=%d err=%v", calls, cleaned, err)
}
}
func TestExecutorsDoNotStartInvalidOrCanceledRequests(t *testing.T) {
canceled, cancel := context.WithCancel(context.Background())
cancel()
for _, test := range []struct {
ctx context.Context
config *Config
options Options[string]
attempt Attempt[string]
}{
{ctx: canceled, options: Options[string]{}, attempt: func(context.Context, string) (string, error) { t.Fatal("called after cancellation"); return "", nil }},
{ctx: context.Background(), options: Options[string]{Kind: 99}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid kind"); return "", nil }},
{ctx: context.Background(), options: Options[string]{Timeout: -time.Second}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid timeout"); return "", nil }},
{ctx: context.Background(), options: Options[string]{}, attempt: nil},
{ctx: context.Background(), config: &Config{Mirrors: []Mirror{{Type: "invalid", URL: "https://mirror.example"}}}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid config"); return "", nil }},
} {
if _, err := Race(test.ctx, test.config, testSource, test.options, test.attempt); err == nil {
t.Fatal("invalid race accepted")
}
if _, err := Fallback(test.ctx, test.config, testSource, test.options, test.attempt); err == nil {
t.Fatal("invalid fallback accepted")
}
}
}
func TestAttemptPanicBecomesError(t *testing.T) {
attempt := func(context.Context, string) (string, error) { panic("broken operation") }
if _, err := Race(context.Background(), nil, testSource, Options[string]{}, attempt); err == nil || !strings.Contains(err.Error(), "broken operation") {
t.Fatal(err)
}
if _, err := Fallback(context.Background(), nil, testSource, Options[string]{}, attempt); err == nil || !strings.Contains(err.Error(), "broken operation") {
t.Fatal(err)
}
}
func TestNonGithubResourcesAndDisabledMirrorsUseOnlyOriginal(t *testing.T) {
for _, test := range []struct {
config *Config
raw string
}{{testConfig(), "https://example.com/resource"}, {&Config{Enabled: false, Mirrors: testConfig().Mirrors}, testSource}} {
calls := 0
got, err := Race(context.Background(), test.config, test.raw, Options[string]{}, func(_ context.Context, address string) (string, error) { calls++; return address, nil })
if err != nil || got != test.raw || calls != 1 {
t.Fatalf("got=%s calls=%d err=%v", got, calls, err)
}
}
}
func TestRepositoryAttemptsHaveNoDefaultDeadline(t *testing.T) {
value, err := Fallback(context.Background(), nil, testSource, Options[string]{Kind: Repository}, func(ctx context.Context, raw string) (string, error) {
if _, ok := ctx.Deadline(); ok {
t.Fatal("repository attempt has a total deadline")
}
return raw, nil
})
if err != nil || value != testSource {
t.Fatalf("value=%s error=%v", value, err)
}
}
+22 -1
View File
@@ -1,8 +1,29 @@
package bt package bt
import (
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
)
type trackerSubscriptionStatus struct {
Success bool `json:"success"`
TrackerCount int `json:"trackerCount"`
LastFetchTime time.Time `json:"lastFetchTime"`
}
type state struct {
TrackerSubscriptionStatuses map[string]trackerSubscriptionStatus `json:"trackerSubscriptionStatuses"`
TrackerSources map[string][]string `json:"trackerSources"`
LastTrackerUpdateTime time.Time `json:"lastTrackerUpdateTime"`
}
type config struct { type config struct {
fetcher.ConfigState[state]
TrackerSubscribeURLs []string `json:"trackerSubscribeUrls"`
AutoUpdateTrackers bool `json:"autoUpdateTrackers"`
CustomTrackers []string `json:"customTrackers"`
ListenPort int `json:"listenPort"` ListenPort int `json:"listenPort"`
Trackers []string `json:"trackers"`
// SeedKeep is always keep seeding after downloading is complete, unless manually stopped. // SeedKeep is always keep seeding after downloading is complete, unless manually stopped.
SeedKeep bool `json:"seedKeep"` SeedKeep bool `json:"seedKeep"`
// SeedRatio is the ratio of uploaded data to downloaded data to seed. // SeedRatio is the ratio of uploaded data to downloaded data to seed.
+18 -2
View File
@@ -198,7 +198,7 @@ func TestBTDownloadSources(t *testing.T) {
} }
func TestBTDirectDownloadWithExtraTrackers(t *testing.T) { func TestBTDirectDownloadWithExtraTrackers(t *testing.T) {
for _, source := range []string{"request", "config"} { for _, source := range []string{"request", "config", "subscription"} {
t.Run(source, func(t *testing.T) { t.Run(source, func(t *testing.T) {
fixture := makeBTFixture(t, map[string][]byte{ fixture := makeBTFixture(t, map[string][]byte{
"a.bin": btPayload(20*1024, "direct-a"), "a.bin": btPayload(20*1024, "direct-a"),
@@ -217,7 +217,23 @@ func TestBTDirectDownloadWithExtraTrackers(t *testing.T) {
if source == "request" { if source == "request" {
req.Extra = &btprotocol.ReqExtra{Trackers: []string{fixture.TrackerURL}} req.Extra = &btprotocol.ReqExtra{Trackers: []string{fixture.TrackerURL}}
} else { } else {
f.config.Trackers = []string{fixture.TrackerURL} backendConfig := new(FetcherManager).DefaultConfig().(*config)
if source == "config" {
backendConfig.CustomTrackers = []string{fixture.TrackerURL}
} else {
backendConfig.TrackerSubscribeURLs = []string{"https://subscription.example/list"}
backendConfig.State.TrackerSources[backendConfig.TrackerSubscribeURLs[0]] = []string{fixture.TrackerURL}
}
f.ctl.GetConfig = func(dst any) {
raw, err := json.Marshal(backendConfig)
if err != nil {
t.Error(err)
return
}
if err := json.Unmarshal(raw, dst); err != nil {
t.Error(err)
}
}
} }
// CreateDirect supplies options before Downloader resolves the task. // CreateDirect supplies options before Downloader resolves the task.
dir := filepath.Join(t.TempDir(), "direct-download") dir := filepath.Join(t.TempDir(), "direct-download")
+9 -4
View File
@@ -744,6 +744,7 @@ func (f *Fetcher) WaitUpload() (err error) {
} }
func (f *Fetcher) addTorrent(req *base.Request) (err error) { func (f *Fetcher) addTorrent(req *base.Request) (err error) {
f.ctl.GetConfig(&f.config)
if err = base.ParseReqExtra[bt.ReqExtra](req); err != nil { if err = base.ParseReqExtra[bt.ReqExtra](req); err != nil {
return return
} }
@@ -816,11 +817,9 @@ func (f *Fetcher) addTorrent(req *base.Request) (err error) {
} }
} }
} }
if len(f.config.Trackers) > 0 { for _, tracker := range f.config.effectiveTrackers() {
for _, tracker := range f.config.Trackers {
trackers[tracker] = true trackers[tracker] = true
} }
}
if len(trackers) > 0 { if len(trackers) > 0 {
for tracker := range trackers { for tracker := range trackers {
spec.Trackers = append(spec.Trackers, []string{tracker}) spec.Trackers = append(spec.Trackers, []string{tracker})
@@ -947,8 +946,14 @@ func (fm *FetcherManager) AutoRename() bool {
func (fm *FetcherManager) DefaultConfig() any { func (fm *FetcherManager) DefaultConfig() any {
return &config{ return &config{
TrackerSubscribeURLs: append([]string{}, defaultTrackerSubscribeURLs...),
AutoUpdateTrackers: true,
CustomTrackers: []string{},
ConfigState: fetcher.ConfigState[state]{State: state{
TrackerSources: map[string][]string{},
TrackerSubscriptionStatuses: map[string]trackerSubscriptionStatus{},
}},
ListenPort: 0, ListenPort: 0,
Trackers: []string{},
SeedKeep: false, SeedKeep: false,
SeedRatio: 1.0, SeedRatio: 1.0,
SeedTime: 120 * 60, SeedTime: 120 * 60,
+1 -1
View File
@@ -227,7 +227,7 @@ func buildConfigFetcher(t testing.TB, proxyConfig *base.DownloaderProxyConfig) f
fetcher := new(FetcherManager).Build() fetcher := new(FetcherManager).Build()
newController := controller.NewController() newController := controller.NewController()
mockCfg := config{ mockCfg := config{
Trackers: []string{ CustomTrackers: []string{
"udp://tracker.birkenwald.de:6969/announce", "udp://tracker.birkenwald.de:6969/announce",
"udp://tracker.bitsearch.to:1337/announce", "udp://tracker.bitsearch.to:1337/announce",
}} }}
+234
View File
@@ -0,0 +1,234 @@
package bt
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/url"
"slices"
"strings"
"sync"
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
)
var defaultTrackerSubscribeURLs = []string{
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_https.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_udp.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ws.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best_ip.txt",
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/all.txt",
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best.txt",
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/http.txt",
}
func validTracker(raw string) bool {
u, err := url.Parse(raw)
if err != nil || u.Hostname() == "" || u.User != nil {
return false
}
switch u.Scheme {
case "udp", "http", "https", "ws", "wss":
return true
}
return false
}
func uniqueTrackers(groups ...[]string) []string {
result := []string{}
seen := map[string]bool{}
for _, group := range groups {
for _, raw := range group {
value := strings.TrimSpace(raw)
if value != "" && !seen[value] {
seen[value] = true
result = append(result, value)
}
}
}
return result
}
// effectiveTrackers derives the client list without persisting duplicate lists.
func (c *config) effectiveTrackers() []string {
groups := [][]string{c.CustomTrackers}
for _, source := range c.TrackerSubscribeURLs {
groups = append(groups, c.State.TrackerSources[source])
}
return uniqueTrackers(groups...)
}
func (fm *FetcherManager) NormalizeConfig(raw json.RawMessage) (any, error) {
c := fm.DefaultConfig().(*config)
if err := json.Unmarshal(raw, c); err != nil {
return nil, err
}
if c.ListenPort < 0 || c.ListenPort > 65535 || c.SeedRatio < 0 || c.SeedTime < 0 {
return nil, errors.New("invalid BT configuration")
}
if len(c.TrackerSubscribeURLs) > 64 {
return nil, errors.New("too many tracker subscriptions")
}
c.TrackerSubscribeURLs = uniqueTrackers(c.TrackerSubscribeURLs)
for _, source := range c.TrackerSubscribeURLs {
u, err := url.Parse(source)
if err != nil || u == nil || (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil {
return nil, errors.New("invalid tracker subscription URL")
}
}
c.CustomTrackers = uniqueTrackers(c.CustomTrackers)
for _, tracker := range c.CustomTrackers {
if !validTracker(tracker) {
return nil, fmt.Errorf("invalid tracker URL: %s", tracker)
}
}
if c.State.TrackerSources == nil {
c.State.TrackerSources = map[string][]string{}
}
for source := range c.State.TrackerSources {
if !slices.Contains(c.TrackerSubscribeURLs, source) {
delete(c.State.TrackerSources, source)
}
}
if c.State.TrackerSubscriptionStatuses == nil {
c.State.TrackerSubscriptionStatuses = map[string]trackerSubscriptionStatus{}
}
for source := range c.State.TrackerSubscriptionStatuses {
if !slices.Contains(c.TrackerSubscribeURLs, source) {
delete(c.State.TrackerSubscriptionStatuses, source)
}
}
return c, nil
}
func (fm *FetcherManager) CronJobs(host fetcher.ProtocolHost) []fetcher.CronJob {
return []fetcher.CronJob{{
Name: "update-trackers",
Every: 24 * time.Hour,
Timeout: 90 * time.Second,
Run: func(ctx context.Context, source fetcher.TriggerSource) error {
return fm.updateTrackers(ctx, host, source)
},
}}
}
func parseTrackers(data []byte) ([]string, error) {
scanner := bufio.NewScanner(bytes.NewReader(data))
scanner.Buffer(make([]byte, 4096), 64*1024)
result := []string{}
for scanner.Scan() {
line := strings.TrimSpace(strings.TrimPrefix(scanner.Text(), "\ufeff"))
if validTracker(line) {
result = append(result, line)
}
if len(result) > 20000 {
return nil, errors.New("too many trackers")
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
if len(result) == 0 {
return nil, errors.New("subscription contains no valid trackers")
}
return uniqueTrackers(result), nil
}
func (fm *FetcherManager) updateTrackers(ctx context.Context, host fetcher.ProtocolHost, source fetcher.TriggerSource) error {
snapshot, err := fetcher.ReadConfig[config](host)
if err != nil {
return err
}
if (source == fetcher.TriggerScheduled && !snapshot.AutoUpdateTrackers) || len(snapshot.TrackerSubscribeURLs) == 0 {
return fetcher.ErrJobSkipped
}
fetched := map[string][]string{}
statuses := map[string]trackerSubscriptionStatus{}
var failures []error
var mu sync.Mutex
var wg sync.WaitGroup
slots := make(chan struct{}, 4)
for _, address := range snapshot.TrackerSubscribeURLs {
wg.Add(1)
go func(address string) {
defer wg.Done()
select {
case slots <- struct{}{}:
defer func() { <-slots }()
case <-ctx.Done():
return
}
data, err := host.Fetch(ctx, address, 2<<20, func(body []byte) error { _, err := parseTrackers(body); return err })
var trackers []string
if err == nil {
trackers, err = parseTrackers(data)
}
mu.Lock()
defer mu.Unlock()
statuses[address] = trackerSubscriptionStatus{Success: err == nil, TrackerCount: len(trackers), LastFetchTime: time.Now().UTC()}
if err != nil {
failures = append(failures, fmt.Errorf("subscription %s: %w", address, err))
} else {
fetched[address] = trackers
}
}(address)
}
wg.Wait()
if err := ctx.Err(); err != nil {
failures = append(failures, err)
}
if len(statuses) == 0 {
return errors.Join(failures...)
}
err = fetcher.UpdateState[config, state](host, func(current *config, state *state) error {
if !slices.Equal(current.TrackerSubscribeURLs, snapshot.TrackerSubscribeURLs) {
return fetcher.ErrConfigChanged
}
if state.TrackerSources == nil {
state.TrackerSources = map[string][]string{}
}
if state.TrackerSubscriptionStatuses == nil {
state.TrackerSubscriptionStatuses = map[string]trackerSubscriptionStatus{}
}
for address, status := range statuses {
state.TrackerSubscriptionStatuses[address] = status
}
for address, trackers := range fetched {
state.TrackerSources[address] = trackers
}
if len(failures) == 0 {
state.LastTrackerUpdateTime = time.Now().UTC()
}
return nil
})
if err != nil {
return err
}
current, err := fetcher.ReadConfig[config](host)
if err != nil {
return err
}
fm.applyTrackers(current.effectiveTrackers())
return errors.Join(failures...)
}
func (fm *FetcherManager) applyTrackers(trackers []string) {
lock.Lock()
defer lock.Unlock()
if client == nil {
return
}
announce := make([][]string, 0, len(trackers))
for _, tracker := range trackers {
announce = append(announce, []string{tracker})
}
for _, t := range client.Torrents() {
info := t.Info()
if info == nil || (info.Private != nil && *info.Private) {
continue
}
t.AddTrackers(announce)
}
}
+217
View File
@@ -0,0 +1,217 @@
package bt
import (
"context"
"encoding/json"
"errors"
"reflect"
"sync"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
)
func trackerTestHost(t *testing.T, c *config, fetch func(context.Context, string, int64, func([]byte) error) ([]byte, error)) (fetcher.ProtocolHost, *sync.Mutex) {
t.Helper()
mu := &sync.Mutex{}
return fetcher.ProtocolHost{
Read: func(dst any) error {
mu.Lock()
defer mu.Unlock()
raw, err := json.Marshal(c)
if err != nil {
return err
}
return json.Unmarshal(raw, dst)
},
UpdateState: func(change func(config, state json.RawMessage) (any, error)) error {
mu.Lock()
defer mu.Unlock()
raw, _ := json.Marshal(c)
currentState, _ := json.Marshal(c.State)
next, err := change(raw, currentState)
if err != nil {
return err
}
raw, err = json.Marshal(next)
if err != nil {
return err
}
return json.Unmarshal(raw, &c.State)
},
Fetch: fetch,
}, mu
}
func TestTrackerRefreshPartialSuccessUsesLatestConfig(t *testing.T) {
fm := &FetcherManager{}
cfg := fm.DefaultConfig().(*config)
cfg.TrackerSubscribeURLs = []string{"https://a.example/list", "https://b.example/list"}
cfg.CustomTrackers = []string{"udp://custom.example:80/announce"}
cfg.State.TrackerSources = map[string][]string{cfg.TrackerSubscribeURLs[0]: {"udp://old.example:80/announce"}, cfg.TrackerSubscribeURLs[1]: {"udp://cached.example:80/announce"}}
host, mu := trackerTestHost(t, cfg, nil)
host.Fetch = func(ctx context.Context, address string, _ int64, validate func([]byte) error) ([]byte, error) {
if address == "https://b.example/list" {
return nil, errors.New("offline")
}
// Simulate a frontend update while the network request is in flight.
mu.Lock()
cfg.CustomTrackers = []string{"udp://new-custom.example:80/announce"}
mu.Unlock()
data := []byte("udp://fresh.example:80/announce\n# ignored\nudp://fresh.example:80/announce\n")
return data, validate(data)
}
err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual)
if err == nil {
t.Fatal("partial failure must be reported")
}
want := []string{"udp://new-custom.example:80/announce", "udp://fresh.example:80/announce", "udp://cached.example:80/announce"}
if !reflect.DeepEqual(cfg.effectiveTrackers(), want) {
t.Fatalf("got %v", cfg.effectiveTrackers())
}
success := cfg.State.TrackerSubscriptionStatuses["https://a.example/list"]
failure := cfg.State.TrackerSubscriptionStatuses["https://b.example/list"]
if !success.Success || success.TrackerCount != 1 || success.LastFetchTime.IsZero() || failure.Success || failure.TrackerCount != 0 || failure.LastFetchTime.IsZero() {
t.Fatalf("unexpected subscription statuses: %+v", cfg.State.TrackerSubscriptionStatuses)
}
if !cfg.State.LastTrackerUpdateTime.IsZero() {
t.Fatal("partial refresh marked fully successful")
}
}
func TestTrackerRefreshDiscardsChangedSubscriptions(t *testing.T) {
fm := &FetcherManager{}
cfg := fm.DefaultConfig().(*config)
cfg.TrackerSubscribeURLs = []string{"https://a.example/list"}
host, mu := trackerTestHost(t, cfg, nil)
host.Fetch = func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
mu.Lock()
cfg.TrackerSubscribeURLs = []string{}
mu.Unlock()
return []byte("udp://fresh.example:80/announce"), nil
}
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); !errors.Is(err, fetcher.ErrConfigChanged) {
t.Fatal(err)
}
if len(cfg.State.TrackerSources) != 0 {
t.Fatal("outdated source was committed")
}
}
func TestTrackerRefreshManualWorksWhenAutomaticDisabled(t *testing.T) {
fm := &FetcherManager{}
cfg := fm.DefaultConfig().(*config)
cfg.AutoUpdateTrackers = false
cfg.TrackerSubscribeURLs = []string{"https://a.example/list"}
calls := 0
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
calls++
return []byte("udp://fresh.example:80/announce"), nil
})
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerScheduled); !errors.Is(err, fetcher.ErrJobSkipped) {
t.Fatal(err)
}
if calls != 0 {
t.Fatal("scheduled job fetched while disabled")
}
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); err != nil {
t.Fatal(err)
}
if calls != 1 || cfg.State.LastTrackerUpdateTime.IsZero() {
t.Fatal("manual refresh did not succeed")
}
}
func TestTrackerConfigPreservesExplicitSettings(t *testing.T) {
value, err := (&FetcherManager{}).NormalizeConfig(json.RawMessage(`{
"listenPort":6881, "trackerSubscribeUrls":[], "autoUpdateTrackers":false,
"customTrackers":["udp://custom.example:80/announce"]
}`))
if err != nil {
t.Fatal(err)
}
cfg := value.(*config)
if cfg.AutoUpdateTrackers || len(cfg.TrackerSubscribeURLs) != 0 || cfg.ListenPort != 6881 {
t.Fatalf("%+v", cfg)
}
if !reflect.DeepEqual(cfg.effectiveTrackers(), cfg.CustomTrackers) {
t.Fatalf("derived trackers: %v", cfg.effectiveTrackers())
}
}
func TestParseTrackerList(t *testing.T) {
got, err := parseTrackers([]byte("\ufeffudp://tracker.example:80/announce\r\n\n<html>error</html>\nhttps://tracker.example/announce\n"))
if err != nil || len(got) != 2 {
t.Fatalf("%v %v", got, err)
}
if _, err := parseTrackers([]byte("<html>mirror error</html>")); err == nil {
t.Fatal("invalid mirror body accepted")
}
}
func TestTrackerDefaultsRegisterDailyJob(t *testing.T) {
jobs := (&FetcherManager{}).CronJobs(fetcher.ProtocolHost{})
if len(jobs) != 1 || jobs[0].Name != "update-trackers" || jobs[0].Every != 24*time.Hour {
t.Fatal(jobs)
}
}
func TestTrackerRefreshAllFailuresPersistStatusesAndKeepCache(t *testing.T) {
fm := &FetcherManager{}
cfg := fm.DefaultConfig().(*config)
address := "https://custom.example/trackers.txt"
cfg.TrackerSubscribeURLs = []string{address}
cfg.State.TrackerSources[address] = []string{"udp://cached.example:80/announce"}
cfg.State.TrackerSubscriptionStatuses[address] = trackerSubscriptionStatus{Success: true, TrackerCount: 1, LastFetchTime: time.Now().Add(-time.Hour)}
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
return nil, errors.New("offline")
})
before := time.Now()
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); err == nil {
t.Fatal("failure was not reported")
}
status, ok := cfg.State.TrackerSubscriptionStatuses[address]
if !ok || status.Success || status.TrackerCount != 0 || status.LastFetchTime.Before(before) {
t.Fatalf("failed fetch status not stored: %+v", status)
}
if !reflect.DeepEqual(cfg.State.TrackerSources[address], []string{"udp://cached.example:80/announce"}) {
t.Fatal("last good cache lost")
}
if !reflect.DeepEqual(cfg.effectiveTrackers(), cfg.State.TrackerSources[address]) {
t.Fatal("cached trackers not retained")
}
}
func TestTrackerConfigPrunesRemovedSubscriptions(t *testing.T) {
fm := &FetcherManager{}
cfg := fm.DefaultConfig().(*config)
address := "https://custom.example/trackers.txt"
cfg.TrackerSubscribeURLs = []string{}
cfg.State.TrackerSources[address] = []string{"udp://cached.example:80/announce"}
cfg.State.TrackerSubscriptionStatuses[address] = trackerSubscriptionStatus{Success: true, TrackerCount: 1, LastFetchTime: time.Now()}
raw, _ := json.Marshal(cfg)
value, err := fm.NormalizeConfig(raw)
if err != nil {
t.Fatal(err)
}
next := value.(*config)
if len(next.State.TrackerSources) != 0 || len(next.State.TrackerSubscriptionStatuses) != 0 || len(next.effectiveTrackers()) != 0 {
t.Fatal("removed subscription still owns cached results")
}
}
func TestTrackerRefreshTimeoutPersistsCompletedAttemptStatus(t *testing.T) {
fm := &FetcherManager{}
cfg := fm.DefaultConfig().(*config)
address := "https://custom.example/trackers.txt"
cfg.TrackerSubscribeURLs = []string{address}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
cancel()
return nil, context.DeadlineExceeded
})
if err := fm.updateTrackers(ctx, host, fetcher.TriggerManual); !errors.Is(err, context.DeadlineExceeded) {
t.Fatal(err)
}
status, ok := cfg.State.TrackerSubscriptionStatuses[address]
if !ok || status.Success || status.TrackerCount != 0 || status.LastFetchTime.IsZero() {
t.Fatalf("timeout status lost: %+v", status)
}
}
+6 -1
View File
@@ -2,6 +2,7 @@ package api
import ( import (
"fmt" "fmt"
"net/url"
pathpkg "path" pathpkg "path"
"strings" "strings"
) )
@@ -75,7 +76,11 @@ func (r RouteSpec) Match(rawPath string) (map[string]string, bool) {
params := make(map[string]string) params := make(map[string]string)
for index, part := range r.parts { for index, part := range r.parts {
if part.param != "" { if part.param != "" {
params[part.param] = segments[index] value, err := url.PathUnescape(segments[index])
if err != nil {
return nil, false
}
params[part.param] = value
continue continue
} }
if part.literal != segments[index] { if part.literal != segments[index] {
+34
View File
@@ -2,7 +2,9 @@ package api
import ( import (
"bytes" "bytes"
"context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"net/http" "net/http"
"net/url" "net/url"
@@ -62,6 +64,8 @@ func NewService(downloader *download.Downloader) (*Service, error) {
newRoute(http.MethodGet, "/api/v1/tasks/{id}/status", service.getTaskStatus), newRoute(http.MethodGet, "/api/v1/tasks/{id}/status", service.getTaskStatus),
newRoute(http.MethodGet, "/api/v1/tasks", service.getTasks), newRoute(http.MethodGet, "/api/v1/tasks", service.getTasks),
newRoute(http.MethodGet, "/api/v1/tasks/{id}/stats", service.getStats), newRoute(http.MethodGet, "/api/v1/tasks/{id}/stats", service.getStats),
newRoute(http.MethodPost, "/api/v1/protocols/{protocol}/jobs/{name}/trigger", service.triggerProtocolJob),
newRoute(http.MethodPost, "/api/v1/mirrors/{provider}/candidates", service.mirrorCandidates),
newRoute(http.MethodGet, "/api/v1/config", service.getConfig), newRoute(http.MethodGet, "/api/v1/config", service.getConfig),
newRoute(http.MethodPut, "/api/v1/config", service.putConfig), newRoute(http.MethodPut, "/api/v1/config", service.putConfig),
newRoute(http.MethodPost, "/api/v1/extensions", service.installExtension), newRoute(http.MethodPost, "/api/v1/extensions", service.installExtension),
@@ -294,6 +298,36 @@ func (s *Service) getStats(ctx *Context) *Response {
return newOKResponse(stats) return newOKResponse(stats)
} }
func (s *Service) triggerProtocolJob(ctx *Context) *Response {
if err := s.Downloader.TriggerProtocolJob(context.Background(), ctx.PathParam("protocol"), ctx.PathParam("name")); err != nil {
status := http.StatusBadRequest
if errors.Is(err, download.ErrUnSupportedProtocol) || errors.Is(err, download.ErrProtocolJobNotFound) {
status = http.StatusNotFound
}
return newStatusResponse(status, restmodel.NewErrorResult(err.Error()))
}
return newNilResponse()
}
func (s *Service) mirrorCandidates(ctx *Context) *Response {
var request restmodel.MirrorCandidatesReq
if response := decodeRequest(ctx.Body, &request); response != nil {
return response
}
urls, err := s.Downloader.GetMirrorCandidates(ctx.PathParam("provider"), request.URL)
if err != nil {
status := http.StatusInternalServerError
switch {
case errors.Is(err, download.ErrMirrorProviderNotFound):
status = http.StatusNotFound
case errors.Is(err, download.ErrInvalidMirrorURL):
status = http.StatusBadRequest
}
return newStatusResponse(status, restmodel.NewErrorResult(err.Error()))
}
return newOKResponse(&restmodel.MirrorCandidatesResp{URLs: urls})
}
func (s *Service) getConfig(_ *Context) *Response { func (s *Service) getConfig(_ *Context) *Response {
config, err := s.Downloader.GetConfig() config, err := s.Downloader.GetConfig()
if err != nil { if err != nil {
+187
View File
@@ -1,6 +1,7 @@
package api package api
import ( import (
"context"
"encoding/json" "encoding/json"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@@ -8,7 +9,10 @@ import (
"path/filepath" "path/filepath"
"reflect" "reflect"
"testing" "testing"
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
"github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download" "github.com/GopeedLab/gopeed/pkg/download"
restmodel "github.com/GopeedLab/gopeed/pkg/rest/model" restmodel "github.com/GopeedLab/gopeed/pkg/rest/model"
@@ -80,3 +84,186 @@ func serviceTestRequest[T any](t *testing.T, service *Service, method, path stri
} }
return result.Data return result.Data
} }
func TestFullConfigDispatch(t *testing.T) {
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&httpProtocol.FetcherManager{}}})
if err := d.Setup(); err != nil {
t.Fatal(err)
}
defer d.Close()
service, err := NewService(d)
if err != nil {
t.Fatal(err)
}
response := service.Dispatch(http.MethodPut, "/api/v1/config", nil, []byte(`{"protocolConfig":{"http":{"connections":3,"useServerCtime":false}}}`), nil)
if response.StatusCode != http.StatusOK {
t.Fatalf("%+v", response)
}
status, body := service.JSON(http.MethodGet, "/api/v1/config", nil, nil, nil)
var result struct {
Data struct {
ProtocolConfig map[string]struct {
Connections int `json:"connections"`
} `json:"protocolConfig"`
} `json:"data"`
}
if err := json.Unmarshal(body, &result); err != nil {
t.Fatal(err)
}
if status != http.StatusOK || result.Data.ProtocolConfig["http"].Connections != 3 {
t.Fatalf("%d %s", status, body)
}
if response := service.Dispatch(http.MethodPost, "/api/v1/protocols/http/jobs/missing/trigger", nil, nil, nil); response.StatusCode != http.StatusNotFound {
t.Fatalf("%+v", response)
}
}
func TestNativePathParametersAreDecodedOnce(t *testing.T) {
route := newRoute(http.MethodGet, "/api/v1/tasks/{id}/status", func(c *Context) *Response { return newOKResponse(c.PathParam("id")) })
for _, test := range []struct{ path, want string }{
{"/api/v1/tasks/%E5%BC%80%E5%8F%91%E8%80%85%40name/status", "开发者@name"},
{"/api/v1/tasks/literal%252Fname/status", "literal%2Fname"},
} {
response := (&Service{routes: []RouteSpec{route}}).Dispatch(http.MethodGet, test.path, nil, nil, nil)
raw, _ := json.Marshal(response.Body)
var body struct {
Data string `json:"data"`
}
_ = json.Unmarshal(raw, &body)
if body.Data != test.want {
t.Fatalf("%s => %q", test.path, body.Data)
}
}
}
func TestMirrorCandidates(t *testing.T) {
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&httpProtocol.FetcherManager{}}})
if err := d.Setup(); err != nil {
t.Fatal(err)
}
defer d.Close()
cfg, _ := d.GetConfig()
cfg.GithubMirror = &base.GithubMirrorConfig{Enabled: true, Mirrors: []base.GithubMirror{
{Type: "jsdelivr", URL: "https://cdn.example/gh"},
{Type: "ghProxy", URL: "https://proxy.example"},
{Type: "ghProxy", URL: "https://proxy.example"},
}}
if err := d.PutConfig(cfg); err != nil {
t.Fatal(err)
}
service, err := NewService(d)
if err != nil {
t.Fatal(err)
}
for _, official := range []string{
"https://github.com/GopeedLab/gopeed/releases/download/v2.0.0/package.zip",
"https://raw.githubusercontent.com/owner/repo/main/trackers.txt",
} {
request, _ := json.Marshal(map[string]string{"url": official})
status, raw := service.JSON(http.MethodPost, "/api/v1/mirrors/github/candidates", nil, request, nil)
var body struct {
Data struct {
URLs []string `json:"urls"`
} `json:"data"`
}
if err := json.Unmarshal(raw, &body); err != nil {
t.Fatal(err)
}
want := 2
if official == "https://raw.githubusercontent.com/owner/repo/main/trackers.txt" {
want = 3
}
urls := body.Data.URLs
if status != http.StatusOK || len(urls) != want || urls[0] != official || urls[len(urls)-1] != "https://proxy.example/"+official {
t.Fatalf("%d %s", status, raw)
}
}
for _, invalid := range []string{"https://api.github.com/repos/GopeedLab/gopeed/releases?per_page=10", "", "https://example.com/x", "http://github.com/x", "https://github.com.evil.test/x", "https://[email protected]/x", "https://github.com:443/x", "https://github.com:/x", "https://github.com/x#fragment", "https://github.com/x#", "/relative", "https://github.com%2fevil.test/x"} {
request, _ := json.Marshal(map[string]string{"url": invalid})
if response := service.Dispatch(http.MethodPost, "/api/v1/mirrors/github/candidates", nil, request, nil); response.StatusCode != http.StatusBadRequest {
t.Fatalf("accepted %s: %+v", invalid, response)
}
}
cfg.GithubMirror.Enabled = false
if err := d.PutConfig(cfg); err != nil {
t.Fatal(err)
}
status, raw := service.JSON(http.MethodPost, "/api/v1/mirrors/github/candidates", nil, []byte(`{"url":"https://github.com/owner/repo"}`), nil)
var body struct {
Data struct {
URLs []string `json:"urls"`
} `json:"data"`
}
if err := json.Unmarshal(raw, &body); err != nil || status != http.StatusOK || len(body.Data.URLs) != 1 || body.Data.URLs[0] != "https://github.com/owner/repo" {
t.Fatalf("%d %s", status, raw)
}
for _, provider := range []string{"unknown", "GitHub"} {
if response := service.Dispatch(http.MethodPost, "/api/v1/mirrors/"+provider+"/candidates", nil, []byte(`{"url":"https://github.com/owner/repo"}`), nil); response.StatusCode != http.StatusNotFound {
t.Fatalf("unknown provider accepted: %+v", response)
}
}
}
type apiJobState struct {
Runs int `json:"runs"`
}
type apiJobConfig struct {
fetcher.ConfigState[apiJobState]
Writable int `json:"writable"`
}
type apiStateJobManager struct{ httpProtocol.FetcherManager }
func (*apiStateJobManager) Name() string { return "state-api" }
func (*apiStateJobManager) DefaultConfig() any { return &apiJobConfig{Writable: 7} }
func (*apiStateJobManager) CronJobs(host fetcher.ProtocolHost) []fetcher.CronJob {
return []fetcher.CronJob{{Name: "refresh", Every: time.Hour, Timeout: time.Second, Run: func(_ context.Context, source fetcher.TriggerSource) error {
if source == fetcher.TriggerScheduled {
return fetcher.ErrJobSkipped
}
return fetcher.UpdateState[apiJobConfig, apiJobState](host, func(cfg *apiJobConfig, state *apiJobState) error { cfg.Writable = 999; state.Runs++; return nil })
}}}
}
func TestFullConfigAndJobTriggerShareProtectedProtocolState(t *testing.T) {
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&apiStateJobManager{}}})
if err := d.Setup(); err != nil {
t.Fatal(err)
}
defer d.Close()
service, err := NewService(d)
if err != nil {
t.Fatal(err)
}
put := []byte(`{"protocolConfig":{"state-api":{"writable":9,"state":"ignore forged state"}}}`)
if response := service.Dispatch(http.MethodPut, "/api/v1/config", nil, put, nil); response.StatusCode != http.StatusOK {
t.Fatalf("%+v", response)
}
if response := service.Dispatch(http.MethodPost, "/api/v1/protocols/state-api/jobs/refresh/trigger", nil, nil, nil); response.StatusCode != http.StatusOK {
t.Fatalf("%+v", response)
}
status, raw := service.JSON(http.MethodGet, "/api/v1/config", nil, nil, nil)
var response struct {
Data struct {
ProtocolConfig map[string]apiJobConfig `json:"protocolConfig"`
} `json:"data"`
}
if err := json.Unmarshal(raw, &response); err != nil {
t.Fatal(err)
}
cfg := response.Data.ProtocolConfig["state-api"]
if status != http.StatusOK || cfg.Writable != 9 || cfg.State.Runs != 1 {
t.Fatalf("%d %s", status, raw)
}
// Replaying an old full PUT must not undo the manual job's state change.
if response := service.Dispatch(http.MethodPut, "/api/v1/config", nil, put, nil); response.StatusCode != http.StatusOK {
t.Fatalf("%+v", response)
}
_, raw = service.JSON(http.MethodGet, "/api/v1/config", nil, nil, nil)
if err := json.Unmarshal(raw, &response); err != nil {
t.Fatal(err)
}
if response.Data.ProtocolConfig["state-api"].State.Runs != 1 {
t.Fatalf("job state overwritten: %s", raw)
}
}
+11
View File
@@ -0,0 +1,11 @@
package base
import "github.com/GopeedLab/gopeed/internal/githubmirror"
// Preserve downloader configuration type names and the existing JSON schema.
type GithubMirror = githubmirror.Mirror
type GithubMirrorConfig = githubmirror.Config
func DefaultGithubMirrorConfig() *GithubMirrorConfig {
return githubmirror.DefaultConfig()
}
+7
View File
@@ -181,6 +181,7 @@ type DownloaderStoreConfig struct {
DownloadDir string `json:"downloadDir"` // DownloadDir is the default directory to save the downloaded files DownloadDir string `json:"downloadDir"` // DownloadDir is the default directory to save the downloaded files
MaxRunning int `json:"maxRunning"` // MaxRunning is the max running download count MaxRunning int `json:"maxRunning"` // MaxRunning is the max running download count
ProtocolConfig map[string]any `json:"protocolConfig"` // ProtocolConfig is special config for each protocol ProtocolConfig map[string]any `json:"protocolConfig"` // ProtocolConfig is special config for each protocol
GithubMirror *GithubMirrorConfig `json:"githubMirror"`
Extra map[string]any `json:"extra"` Extra map[string]any `json:"extra"`
Proxy *DownloaderProxyConfig `json:"proxy"` Proxy *DownloaderProxyConfig `json:"proxy"`
Webhook *WebhookConfig `json:"webhook"` // Webhook is the webhook configuration Webhook *WebhookConfig `json:"webhook"` // Webhook is the webhook configuration
@@ -199,6 +200,9 @@ func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
if cfg.ProtocolConfig == nil { if cfg.ProtocolConfig == nil {
cfg.ProtocolConfig = make(map[string]any) cfg.ProtocolConfig = make(map[string]any)
} }
if cfg.GithubMirror == nil {
cfg.GithubMirror = DefaultGithubMirrorConfig()
}
if cfg.Proxy == nil { if cfg.Proxy == nil {
cfg.Proxy = &DownloaderProxyConfig{} cfg.Proxy = &DownloaderProxyConfig{}
} }
@@ -239,6 +243,9 @@ func (cfg *DownloaderStoreConfig) Merge(beforeCfg *DownloaderStoreConfig) *Downl
if cfg.Extra == nil { if cfg.Extra == nil {
cfg.Extra = beforeCfg.Extra cfg.Extra = beforeCfg.Extra
} }
if cfg.GithubMirror == nil {
cfg.GithubMirror = beforeCfg.GithubMirror
}
if cfg.Proxy == nil { if cfg.Proxy == nil {
cfg.Proxy = beforeCfg.Proxy cfg.Proxy = beforeCfg.Proxy
} }
+21
View File
@@ -15,6 +15,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
"Init", "Init",
&DownloaderStoreConfig{}, &DownloaderStoreConfig{},
&DownloaderStoreConfig{ &DownloaderStoreConfig{
GithubMirror: DefaultGithubMirrorConfig(),
MaxRunning: 5, MaxRunning: 5,
ProtocolConfig: map[string]any{}, ProtocolConfig: map[string]any{},
Proxy: &DownloaderProxyConfig{}, Proxy: &DownloaderProxyConfig{},
@@ -36,6 +37,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
MaxRunning: 10, MaxRunning: 10,
}, },
&DownloaderStoreConfig{ &DownloaderStoreConfig{
GithubMirror: DefaultGithubMirrorConfig(),
MaxRunning: 10, MaxRunning: 10,
ProtocolConfig: map[string]any{}, ProtocolConfig: map[string]any{},
Proxy: &DownloaderProxyConfig{}, Proxy: &DownloaderProxyConfig{},
@@ -59,6 +61,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
}, },
}, },
&DownloaderStoreConfig{ &DownloaderStoreConfig{
GithubMirror: DefaultGithubMirrorConfig(),
MaxRunning: 5, MaxRunning: 5,
ProtocolConfig: map[string]any{ ProtocolConfig: map[string]any{
"key": "value", "key": "value",
@@ -84,6 +87,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
}, },
}, },
&DownloaderStoreConfig{ &DownloaderStoreConfig{
GithubMirror: DefaultGithubMirrorConfig(),
MaxRunning: 5, MaxRunning: 5,
ProtocolConfig: map[string]any{}, ProtocolConfig: map[string]any{},
Proxy: &DownloaderProxyConfig{ Proxy: &DownloaderProxyConfig{
@@ -110,6 +114,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
}, },
}, },
&DownloaderStoreConfig{ &DownloaderStoreConfig{
GithubMirror: DefaultGithubMirrorConfig(),
MaxRunning: 5, MaxRunning: 5,
ProtocolConfig: map[string]any{}, ProtocolConfig: map[string]any{},
Proxy: &DownloaderProxyConfig{}, Proxy: &DownloaderProxyConfig{},
@@ -134,6 +139,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
}, },
}, },
&DownloaderStoreConfig{ &DownloaderStoreConfig{
GithubMirror: DefaultGithubMirrorConfig(),
MaxRunning: 5, MaxRunning: 5,
ProtocolConfig: map[string]any{}, ProtocolConfig: map[string]any{},
Proxy: &DownloaderProxyConfig{}, Proxy: &DownloaderProxyConfig{},
@@ -149,6 +155,20 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
}, },
}, },
}, },
{
name: "Init explicitly disabled mirrors with empty list",
fields: &DownloaderStoreConfig{GithubMirror: &GithubMirrorConfig{Enabled: false, Mirrors: []GithubMirror{}}},
want: &DownloaderStoreConfig{
MaxRunning: 5,
ProtocolConfig: map[string]any{},
GithubMirror: &GithubMirrorConfig{Enabled: false, Mirrors: []GithubMirror{}},
Proxy: &DownloaderProxyConfig{},
Webhook: &WebhookConfig{},
Script: &ScriptConfig{},
AutoTorrent: &AutoTorrentConfig{},
Archive: &ArchiveConfig{},
},
},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
@@ -157,6 +177,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
DownloadDir: tt.fields.DownloadDir, DownloadDir: tt.fields.DownloadDir,
MaxRunning: tt.fields.MaxRunning, MaxRunning: tt.fields.MaxRunning,
ProtocolConfig: tt.fields.ProtocolConfig, ProtocolConfig: tt.fields.ProtocolConfig,
GithubMirror: tt.fields.GithubMirror,
Extra: tt.fields.Extra, Extra: tt.fields.Extra,
Proxy: tt.fields.Proxy, Proxy: tt.fields.Proxy,
Webhook: tt.fields.Webhook, Webhook: tt.fields.Webhook,
+245
View File
@@ -0,0 +1,245 @@
package download
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
btProtocol "github.com/GopeedLab/gopeed/internal/protocol/bt"
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
"github.com/GopeedLab/gopeed/pkg/base"
)
type testJobManager struct {
httpProtocol.FetcherManager
job fetcher.CronJob
}
func (m *testJobManager) Name() string { return "test" }
func (m *testJobManager) CronJobs(fetcher.ProtocolHost) []fetcher.CronJob {
return []fetcher.CronJob{m.job}
}
func testBackendDownloader(t *testing.T, managers ...fetcher.FetcherManager) *Downloader {
t.Helper()
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: managers})
if err := d.Setup(); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := d.Close(); err != nil {
t.Error(err)
}
})
return d
}
func TestProtocolJobCoalescesAndCancelsOnClose(t *testing.T) {
entered := make(chan struct{})
var calls atomic.Int32
manager := &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Minute, Run: func(ctx context.Context, _ fetcher.TriggerSource) error {
if calls.Add(1) == 1 {
close(entered)
}
<-ctx.Done()
return ctx.Err()
}}}
d := testBackendDownloader(t, manager)
result := make(chan error, 2)
go func() { result <- d.TriggerProtocolJob(context.Background(), "test", "refresh") }()
<-entered
waiterCtx, cancel := context.WithCancel(context.Background())
// A canceled waiter joins the existing work without canceling it.
cancel()
if err := d.TriggerProtocolJob(waiterCtx, "test", "refresh"); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if calls.Load() != 1 {
t.Fatalf("executed %d times", calls.Load())
}
if err := d.Close(); err != nil {
t.Fatal(err)
}
if err := <-result; !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if err := d.TriggerProtocolJob(context.Background(), "test", "missing"); !errors.Is(err, ErrProtocolJobNotFound) {
t.Fatal(err)
}
}
func TestProtocolJobPersistsSuccess(t *testing.T) {
manager := &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Minute, Run: func(context.Context, fetcher.TriggerSource) error { return nil }}}
d := testBackendDownloader(t, manager)
if err := d.TriggerProtocolJob(context.Background(), "test", "refresh"); err != nil {
t.Fatal(err)
}
var state protocolJobState
exists, err := d.storage.Get(bucketProtocolJobs, "test/refresh", &state)
if err != nil || !exists || state.LastSuccess.IsZero() {
t.Fatalf("state %+v: %v", state, err)
}
}
func TestRaceResourceValidatesResponsesAndCancelsLosers(t *testing.T) {
loserCanceled := make(chan struct{})
loserStarted := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodConnect:
http.Error(w, "official unavailable", http.StatusBadGateway)
case strings.HasPrefix(r.URL.Path, "/bad/"):
io.WriteString(w, "<html>proxy landing page</html>")
case strings.HasPrefix(r.URL.Path, "/good/"):
<-loserStarted
io.WriteString(w, `{"ok":true}`)
case strings.HasPrefix(r.URL.Path, "/slow/"):
close(loserStarted)
<-r.Context().Done()
close(loserCanceled)
default:
http.NotFound(w, r)
}
}))
defer server.Close()
d := testBackendDownloader(t, &httpProtocol.FetcherManager{})
cfg, _ := d.GetConfig()
cfg.Proxy = &base.DownloaderProxyConfig{Enable: true, Scheme: "http", Host: strings.TrimPrefix(server.URL, "http://")}
cfg.GithubMirror = &base.GithubMirrorConfig{Enabled: true, Mirrors: []base.GithubMirror{
{Type: "ghProxy", URL: server.URL + "/bad"},
{Type: "ghProxy", URL: server.URL + "/slow"},
{Type: "ghProxy", URL: server.URL + "/good"},
}}
if err := d.PutConfig(cfg); err != nil {
t.Fatal(err)
}
body, err := d.fetchResource(context.Background(), "https://raw.githubusercontent.com/owner/repo/main/test.json", 1024, false, func(body []byte) error {
if !json.Valid(body) {
return errors.New("invalid JSON")
}
return nil
})
if err != nil || string(body) != `{"ok":true}` {
t.Fatalf("%s %v", body, err)
}
select {
case <-loserCanceled:
case <-time.After(time.Second):
t.Fatal("losing request was not canceled")
}
}
func TestProtocolConfigInitializesCurrentSchema(t *testing.T) {
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&httpProtocol.FetcherManager{}}})
d.cfg.DownloaderStoreConfig = &base.DownloaderStoreConfig{
GithubMirror: &base.GithubMirrorConfig{Enabled: false, Mirrors: []base.GithubMirror{}},
ProtocolConfig: map[string]any{"http": map[string]any{"connections": 3}},
}
if err := d.storage.Setup([]string{bucketConfig}); err != nil {
t.Fatal(err)
}
defer d.storage.Close()
if err := d.initProtocolConfigs(); err != nil {
t.Fatal(err)
}
if d.cfg.GithubMirror.Enabled || len(d.cfg.GithubMirror.Mirrors) != 0 {
t.Fatal("explicit disabled/empty mirrors lost")
}
value, err := d.readProtocolConfig("http")
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(value)
var cfg struct {
Connections int `json:"connections"`
}
if err := json.Unmarshal(raw, &cfg); err != nil || cfg.Connections != 3 {
t.Fatalf("current protocol config: %s %v", raw, err)
}
if err := d.initProtocolConfigs(); err != nil {
t.Fatal(err)
}
}
func TestProtocolConfigDoesNotImportOldExtraSettings(t *testing.T) {
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&btProtocol.FetcherManager{}}})
d.cfg.DownloaderStoreConfig = &base.DownloaderStoreConfig{
Extra: map[string]any{
"githubMirror": "obsolete invalid value",
"bt": map[string]any{"customTrackers": []string{"udp://old.example:80/announce"}, "autoUpdateTrackers": true},
},
ProtocolConfig: map[string]any{"bt": map[string]any{"autoUpdateTrackers": false, "listenPort": 6881}},
}
if err := d.storage.Setup([]string{bucketConfig}); err != nil {
t.Fatal(err)
}
defer d.storage.Close()
if err := d.initProtocolConfigs(); err != nil {
t.Fatal(err)
}
if !d.cfg.GithubMirror.Enabled || len(d.cfg.GithubMirror.Mirrors) != 4 {
t.Fatal("missing root mirror config must use new defaults")
}
value, err := d.readProtocolConfig("bt")
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(value)
var cfg struct {
ListenPort int `json:"listenPort"`
AutoUpdateTrackers bool `json:"autoUpdateTrackers"`
CustomTrackers []string `json:"customTrackers"`
}
if err := json.Unmarshal(raw, &cfg); err != nil || cfg.ListenPort != 6881 || cfg.AutoUpdateTrackers || len(cfg.CustomTrackers) != 0 {
t.Fatalf("old extra settings affected protocol config: %s %v", raw, err)
}
}
func TestProtocolJobsRunAutomaticallyAndRestoreSuccess(t *testing.T) {
dir := t.TempDir()
executed := make(chan fetcher.TriggerSource, 4)
manager := func() *testJobManager {
return &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Second, Run: func(_ context.Context, source fetcher.TriggerSource) error { executed <- source; return nil }}}
}
makeDownloader := func() *Downloader {
d := NewDownloader(&DownloaderConfig{Storage: NewBoltStorage(dir), StorageDir: dir, TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{manager()}})
if err := d.Setup(); err != nil {
t.Fatal(err)
}
return d
}
d := makeDownloader()
select {
case source := <-executed:
if source != fetcher.TriggerScheduled {
t.Fatal(source)
}
case <-time.After(3 * time.Second):
t.Fatal("overdue job was not run")
}
entry := d.protocolJobs.jobs["test/refresh"]
entry.mu.Lock()
run := entry.active
entry.mu.Unlock()
if run != nil {
<-run.done
}
if err := d.Close(); err != nil {
t.Fatal(err)
}
d = makeDownloader()
defer d.Close()
state := d.protocolJobs.jobs["test/refresh"].state
if state.LastSuccess.IsZero() {
t.Fatal("successful execution not restored")
}
select {
case <-executed:
t.Fatal("recently successful job ran again on restart")
case <-time.After(30 * time.Millisecond):
}
}
+79 -10
View File
@@ -2,6 +2,7 @@ package download
import ( import (
"bytes" "bytes"
"context"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@@ -42,6 +43,7 @@ const (
bucketTaskStats = "task_stats" bucketTaskStats = "task_stats"
// protocol-level shared client state bucket // protocol-level shared client state bucket
bucketProtocolState = "protocol_state" bucketProtocolState = "protocol_state"
bucketProtocolJobs = "protocol_jobs"
// downloader config bucket // downloader config bucket
bucketConfig = "config" bucketConfig = "config"
// downloader extension bucket // downloader extension bucket
@@ -99,6 +101,9 @@ type Progress struct {
} }
type Downloader struct { type Downloader struct {
backgroundCtx context.Context
cancelBackground context.CancelFunc
protocolJobs *protocolScheduler
tempFiles *tempfiles.Scope tempFiles *tempfiles.Scope
webviewProfilesLock sync.Mutex webviewProfilesLock sync.Mutex
webviewProfiles map[string]*extensionWebViewProfile webviewProfiles map[string]*extensionWebViewProfile
@@ -126,6 +131,8 @@ type Downloader struct {
claimedExtractions sync.Map claimedExtractions sync.Map
extensions []*Extension extensions []*Extension
extensionGitLock sync.Mutex
extensionGitWG sync.WaitGroup
blob *internalblob.Registry blob *internalblob.Registry
} }
@@ -135,7 +142,10 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
} }
cfg.Init() cfg.Init()
backgroundCtx, cancelBackground := context.WithCancel(context.Background())
d := &Downloader{ d := &Downloader{
backgroundCtx: backgroundCtx,
cancelBackground: cancelBackground,
cfg: cfg, cfg: cfg,
fetcherCache: make(map[string]fetcher.Fetcher), fetcherCache: make(map[string]fetcher.Fetcher),
waitTasks: make([]*Task, 0), waitTasks: make([]*Task, 0),
@@ -165,7 +175,7 @@ func (d *Downloader) Setup() error {
d.blob = internalblob.NewRegistry("") d.blob = internalblob.NewRegistry("")
// setup storage // setup storage
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketTaskStats, bucketProtocolState, bucketConfig, bucketExtension, bucketExtensionStorage}); err != nil { if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketTaskStats, bucketProtocolState, bucketProtocolJobs, bucketConfig, bucketExtension, bucketExtensionStorage}); err != nil {
return err return err
} }
// load config from storage // load config from storage
@@ -181,14 +191,13 @@ func (d *Downloader) Setup() error {
FirstLoad: true, FirstLoad: true,
} }
} }
// init default config // Initialize the current backend configuration schema and defaults.
d.cfg.DownloaderStoreConfig.Init() if err := d.initProtocolConfigs(); err != nil {
// init protocol config, if not exist, use default config return err
}
// Attach optional protocol state stores after configuration initialization.
for _, fm := range d.cfg.FetchManagers { for _, fm := range d.cfg.FetchManagers {
protocol := fm.Name() protocol := fm.Name()
if _, ok := d.cfg.DownloaderStoreConfig.ProtocolConfig[protocol]; !ok {
d.cfg.DownloaderStoreConfig.ProtocolConfig[protocol] = fm.DefaultConfig()
}
if sfm, ok := fm.(fetcher.StatefulFetcherManager); ok { if sfm, ok := fm.(fetcher.StatefulFetcherManager); ok {
sfm.SetStateStore(&protocolStateStore{ sfm.SetStateStore(&protocolStateStore{
storage: d.storage, storage: d.storage,
@@ -197,6 +206,10 @@ func (d *Downloader) Setup() error {
} }
} }
if err := d.setupProtocolJobs(); err != nil {
return err
}
// load tasks from storage // load tasks from storage
var tasks []*Task var tasks []*Task
if err = d.storage.List(bucketTask, &tasks); err != nil { if err = d.storage.List(bucketTask, &tasks); err != nil {
@@ -365,6 +378,7 @@ func (d *Downloader) Setup() error {
time.Sleep(time.Millisecond * time.Duration(d.cfg.RefreshInterval)) time.Sleep(time.Millisecond * time.Duration(d.cfg.RefreshInterval))
} }
}() }()
d.protocolJobs.start()
return nil return nil
} }
@@ -1181,7 +1195,14 @@ func (d *Downloader) doDelete(task *Task, force bool) (err error) {
} }
func (d *Downloader) Close() error { func (d *Downloader) Close() error {
d.closed.Store(true) if d.closed.Swap(true) {
return nil
}
d.cancelBackground()
d.waitForExtensionGit()
if d.protocolJobs != nil {
d.protocolJobs.close()
}
d.webviewProxyLock.Lock() d.webviewProxyLock.Lock()
if d.webviewProxy != nil { if d.webviewProxy != nil {
_ = d.webviewProxy.Close() _ = d.webviewProxy.Close()
@@ -1336,7 +1357,7 @@ func (d *Downloader) GetTasksByFilter(filter *TaskFilter) []*Task {
func (d *Downloader) GetConfig() (*base.DownloaderStoreConfig, error) { func (d *Downloader) GetConfig() (*base.DownloaderStoreConfig, error) {
d.configLock.RLock() d.configLock.RLock()
defer d.configLock.RUnlock() defer d.configLock.RUnlock()
return d.cfg.DownloaderStoreConfig, nil return util.DeepClone(d.cfg.DownloaderStoreConfig), nil
} }
func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error { func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error {
@@ -1344,12 +1365,60 @@ func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error {
return errors.New("config is nil") return errors.New("config is nil")
} }
next := util.DeepClone(v) next := util.DeepClone(v)
changedProtocols := []string{}
d.configLock.Lock() d.configLock.Lock()
defer d.configLock.Unlock() if d.closed.Load() {
d.configLock.Unlock()
return errors.New("downloader is closed")
}
if next.GithubMirror == nil {
next.GithubMirror = util.DeepClone(d.cfg.GithubMirror)
}
if err := next.GithubMirror.Validate(); err != nil {
d.configLock.Unlock()
return err
}
if next.ProtocolConfig == nil {
next.ProtocolConfig = map[string]any{}
}
for name := range next.ProtocolConfig {
if _, err := d.protocolManager(name); err != nil {
d.configLock.Unlock()
return err
}
}
for _, fm := range d.cfg.FetchManagers {
name := fm.Name()
value, exists := next.ProtocolConfig[name]
if !exists {
value = d.cfg.ProtocolConfig[name]
}
value, err := preserveProtocolState(value, d.cfg.ProtocolConfig[name])
if err == nil {
value, err = normalizeProtocol(fm, value)
}
if err != nil {
d.configLock.Unlock()
return err
}
next.ProtocolConfig[name] = value
oldJSON, _ := json.Marshal(d.cfg.ProtocolConfig[name])
newJSON, _ := json.Marshal(value)
if !bytes.Equal(oldJSON, newJSON) {
changedProtocols = append(changedProtocols, name)
}
}
if err := d.storage.Put(bucketConfig, "config", next); err != nil { if err := d.storage.Put(bucketConfig, "config", next); err != nil {
d.configLock.Unlock()
return err return err
} }
d.cfg.DownloaderStoreConfig = next d.cfg.DownloaderStoreConfig = next
d.configLock.Unlock()
if d.protocolJobs != nil {
for _, name := range changedProtocols {
d.protocolJobs.changed(name)
}
}
return nil return nil
} }
+21 -3
View File
@@ -1241,7 +1241,7 @@ func TestDownloader_Protocol_Config(t *testing.T) {
"trackerSubscribeUrls": []string{ "trackerSubscribeUrls": []string{
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best.txt", "https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best.txt",
}, },
"trackers": []string{ "customTrackers": []string{
"udp://tracker.coppersurfer.tk:6969/announce", "udp://tracker.coppersurfer.tk:6969/announce",
"udp://tracker.leechers-paradise.org:6969/announce", "udp://tracker.leechers-paradise.org:6969/announce",
}, },
@@ -1261,8 +1261,20 @@ func TestDownloader_Protocol_Config(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
if !test.JsonEqual(storeCfg, newStoreCfg) { if newStoreCfg.DownloadDir != storeCfg.DownloadDir || newStoreCfg.Extra["theme"] != "dark" {
t.Errorf("GetConfig() got = %v, want %v", test.ToJson(storeCfg), test.ToJson(newStoreCfg)) t.Fatalf("downloader config was not saved: %+v", newStoreCfg)
}
var savedHTTP struct {
Connections int `json:"connections"`
}
if err := util.MapToStruct(newStoreCfg.ProtocolConfig["http"], &savedHTTP); err != nil || savedHTTP.Connections != 4 {
t.Fatalf("HTTP config: %+v %v", savedHTTP, err)
}
var savedBT struct {
CustomTrackers []string `json:"customTrackers"`
}
if err := util.MapToStruct(newStoreCfg.ProtocolConfig["bt"], &savedBT); err != nil || len(savedBT.CustomTrackers) != 2 {
t.Fatalf("BT derived trackers: %+v %v", savedBT, err)
} }
} }
@@ -2002,6 +2014,9 @@ func TestDownloader_ContinueOnStartupUsesContinueAllSemantics(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
cfg.Extra = map[string]any{"autoStartTasks": true} cfg.Extra = map[string]any{"autoStartTasks": true}
if err := downloader.PutConfig(cfg); err != nil {
t.Fatal(err)
}
if err := downloader.ContinueOnStartup(); err != nil { if err := downloader.ContinueOnStartup(); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -2010,6 +2025,9 @@ func TestDownloader_ContinueOnStartupUsesContinueAllSemantics(t *testing.T) {
} }
cfg.AutoStartTasks = true cfg.AutoStartTasks = true
if err := downloader.PutConfig(cfg); err != nil {
t.Fatal(err)
}
if err := downloader.ContinueOnStartup(); err != nil { if err := downloader.ContinueOnStartup(); err != nil {
t.Fatal(err) t.Fatal(err)
} }
+72 -35
View File
@@ -1,17 +1,19 @@
package download package download
import ( import (
"context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"net/url"
"os" "os"
"path"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/githubmirror"
"github.com/GopeedLab/gopeed/internal/logger" "github.com/GopeedLab/gopeed/internal/logger"
"github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download/engine" "github.com/GopeedLab/gopeed/pkg/download/engine"
@@ -196,50 +198,85 @@ func (d *Downloader) getExtension(identity string) *Extension {
return nil return nil
} }
func (d *Downloader) fetchExtensionByGit(url string, handler func(tempExtPath string, devMode bool) (*Extension, error)) (*Extension, error) { // waitForExtensionGit waits for clones and their install/check handlers before
if !strings.HasPrefix(url, "https://") && !strings.HasPrefix(url, "http://") { // shutdown releases storage and temporary files. Close cancels backgroundCtx first.
url = "https://" + url func (d *Downloader) waitForExtensionGit() {
d.extensionGitLock.Lock()
d.extensionGitLock.Unlock()
d.extensionGitWG.Wait()
} }
// resolve project path func (d *Downloader) fetchExtensionByGit(rawURL string, handler func(string, bool) (*Extension, error)) (*Extension, error) {
parentPath, projectPath := path.Split(url) // Pair admission with shutdown so no operation is added after its wait starts.
// resolve project name and sub path d.extensionGitLock.Lock()
pathArr := strings.SplitN(projectPath, "#", 2) if d.closed.Load() {
projectPath = strings.TrimSuffix(pathArr[0], gitSuffix) d.extensionGitLock.Unlock()
subPath := "" return nil, fmt.Errorf("downloader is closed")
if len(pathArr) > 1 {
subPath = pathArr[1]
} }
d.extensionGitWG.Add(1)
// Use unique suffix to avoid concurrent conflicts when multiple git operations d.extensionGitLock.Unlock()
// target the same extension (e.g., install and update check happening simultaneously) defer d.extensionGitWG.Done()
tempExtDir, err := os.MkdirTemp(d.cfg.TempDir, "gopeed-extension-") ctx := d.backgroundCtx
if !strings.HasPrefix(rawURL, "https://") && !strings.HasPrefix(rawURL, "http://") {
rawURL = "https://" + rawURL
}
parsed, err := url.Parse(rawURL)
if err != nil || parsed.Host == "" {
return nil, fmt.Errorf("invalid extension repository URL")
}
subPath := filepath.FromSlash(parsed.Fragment)
if filepath.IsAbs(subPath) || subPath == ".." || strings.HasPrefix(filepath.Clean(subPath), ".."+string(filepath.Separator)) {
return nil, fmt.Errorf("invalid extension directory")
}
parsed.Fragment = ""
parsed.Path = strings.TrimSuffix(strings.TrimRight(parsed.Path, "/"), gitSuffix) + gitSuffix
officialURL := parsed.String()
cfg, err := d.GetConfig()
if err != nil { if err != nil {
return nil, err return nil, err
} }
if err := d.tempFiles.Track(tempExtDir); err != nil {
return nil, err
}
defer d.tempFiles.Remove(tempExtDir)
proxyOptions := transport.ProxyOptions{} proxyOptions := transport.ProxyOptions{}
proxyUrl := d.cfg.DownloaderStoreConfig.Proxy.ToUrl() if proxyURL := cfg.Proxy.ToUrl(); proxyURL != nil {
if proxyUrl != nil { proxyOptions.URL = proxyURL.Scheme + "://" + proxyURL.Host
proxyOptions.URL = proxyUrl.Scheme + "://" + proxyUrl.Host if proxyURL.User != nil {
proxyOptions.Username = proxyUrl.User.Username() proxyOptions.Username = proxyURL.User.Username()
proxyOptions.Password, _ = proxyUrl.User.Password() proxyOptions.Password, _ = proxyURL.User.Password()
} }
// clone project to extension temp dir }
gitUrl := parentPath + projectPath + gitSuffix tempDir, err := os.MkdirTemp(d.cfg.TempDir, "gopeed-extension-")
if _, err := git.PlainClone(tempExtDir, false, &git.CloneOptions{ if err != nil {
URL: gitUrl,
Depth: 1,
ProxyOptions: proxyOptions,
}); err != nil {
return nil, err return nil, err
} }
if err = d.tempFiles.Track(tempDir); err != nil {
return handler(filepath.Join(tempExtDir, subPath), false) _ = os.RemoveAll(tempDir)
return nil, err
}
defer d.tempFiles.Remove(tempDir)
repoDir := filepath.Join(tempDir, "repo")
repo, err := githubmirror.Fallback(ctx, cfg.GithubMirror, officialURL,
githubmirror.Options[*git.Repository]{Kind: githubmirror.Repository},
func(ctx context.Context, candidate string) (*git.Repository, error) {
if err := os.RemoveAll(repoDir); err != nil {
return nil, err
}
return git.PlainCloneContext(ctx, repoDir, false, &git.CloneOptions{URL: candidate, Depth: 1, ProxyOptions: proxyOptions})
},
)
if err != nil {
return nil, err
}
// Preserve the canonical source instead of persisting a selected mirror.
repoConfig, err := repo.Config()
if err != nil {
return nil, err
}
if remote := repoConfig.Remotes["origin"]; remote != nil {
remote.URLs = []string{officialURL}
}
if err = repo.SetConfig(repoConfig); err != nil {
return nil, err
}
return handler(filepath.Join(repoDir, subPath), false)
} }
func (d *Downloader) parseExtensionByPath(path string) (*Extension, error) { func (d *Downloader) parseExtensionByPath(path string) (*Extension, error) {
+210
View File
@@ -0,0 +1,210 @@
package download
import (
"encoding/json"
"io"
"net/http"
"net/http/cgi"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
"github.com/GopeedLab/gopeed/pkg/base"
)
// Exercise real Git clone/install/check/update without depending on GitHub or
// public mirror availability. The proxy deliberately rejects the official URL.
// github.com/gopeed-test/repo is a placeholder: the successful mirror serves
// the temporary local repository through git http-backend instead.
func newExtensionGitFixture(t *testing.T) (*Downloader, func(string), *atomic.Int32, *atomic.Int32) {
t.Helper()
git, err := exec.LookPath("git")
if err != nil {
t.Skip("Git executable required for HTTP Git fixture")
}
root := t.TempDir()
bare := filepath.Join(root, "repo.git")
work := t.TempDir()
command := func(args ...string) {
t.Helper()
cmd := exec.Command(git, args...)
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("git %v: %v: %s", args, err, output)
}
}
command("-c", "init.defaultBranch=main", "init", "--bare", bare)
command("clone", bare, work)
command("-C", work, "config", "user.name", "Gopeed Test")
command("-C", work, "config", "user.email", "[email protected]")
publish := func(version string) {
t.Helper()
directory := filepath.Join(work, "nested", "extension")
if err := os.MkdirAll(directory, 0700); err != nil {
t.Fatal(err)
}
manifest := map[string]any{"name": "mirror-test", "author": "test", "title": "Mirror test", "version": version, "repository": map[string]string{"url": "https://github.com/gopeed-test/repo", "directory": "nested/extension"}}
body, _ := json.Marshal(manifest)
if err := os.WriteFile(filepath.Join(directory, "manifest.json"), body, 0600); err != nil {
t.Fatal(err)
}
command("-C", work, "add", ".")
command("-C", work, "commit", "-m", version)
command("-C", work, "push", "origin", "HEAD:main")
}
publish("1.0.0")
var official, badMirror atomic.Int32
backend := &cgi.Handler{Path: git, Args: []string{"http-backend"}, Env: []string{"GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}}
mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r.URL.Path = strings.TrimPrefix(r.URL.Path, "/https://github.com/gopeed-test")
r.RequestURI = r.URL.RequestURI()
backend.ServeHTTP(w, r)
}))
t.Cleanup(mirror.Close)
failed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { badMirror.Add(1); http.Error(w, "mirror offline", 502) }))
t.Cleanup(failed.Close)
direct := &http.Transport{}
t.Cleanup(direct.CloseIdleConnections)
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodConnect || r.URL.Hostname() == "github.com" {
official.Add(1)
http.Error(w, "official unavailable", 502)
return
}
request := r.Clone(r.Context())
request.RequestURI = ""
response, err := direct.RoundTrip(request)
if err != nil {
http.Error(w, err.Error(), 502)
return
}
defer response.Body.Close()
for key, values := range response.Header {
for _, value := range values {
w.Header().Add(key, value)
}
}
w.WriteHeader(response.StatusCode)
_, _ = io.Copy(w, response.Body)
}))
t.Cleanup(proxy.Close)
d := testBackendDownloader(t, &httpProtocol.FetcherManager{})
config, _ := d.GetConfig()
config.Proxy = &base.DownloaderProxyConfig{Enable: true, Scheme: "http", Host: strings.TrimPrefix(proxy.URL, "http://")}
config.GithubMirror = &base.GithubMirrorConfig{Enabled: true, Mirrors: []base.GithubMirror{{Type: "ghProxy", URL: failed.URL}, {Type: "ghProxy", URL: mirror.URL}}}
if err := d.PutConfig(config); err != nil {
t.Fatal(err)
}
return d, publish, &official, &badMirror
}
func testInstallExtensionGitURL(t *testing.T, raw string) {
d, _, official, badMirror := newExtensionGitFixture(t)
extension, err := d.InstallExtensionByGit(raw)
if err != nil {
t.Fatal(err)
}
if extension.Version != "1.0.0" || extension.Identity != "test@mirror-test" {
t.Fatalf("%+v", extension)
}
if official.Load() == 0 || badMirror.Load() == 0 {
t.Fatal("ordered fallback was not exercised")
}
if _, err := os.Stat(filepath.Join(d.ExtensionPath(extension), "manifest.json")); err != nil {
t.Fatal(err)
}
}
func TestExtensionGitMirrorInstallAndUpdate(t *testing.T) {
d, publish, official, badMirror := newExtensionGitFixture(t)
extension, err := d.InstallExtensionByGit("https://github.com/gopeed-test/repo#nested/extension")
if err != nil {
t.Fatal(err)
}
publish("1.1.0")
version, err := d.UpgradeCheckExtension(extension.Identity)
if err != nil || version != "1.1.0" {
t.Fatalf("%s %v", version, err)
}
if err = d.UpgradeExtension(extension.Identity); err != nil {
t.Fatal(err)
}
updated, err := d.GetExtension(extension.Identity)
if err != nil || updated.Version != "1.1.0" {
t.Fatalf("%+v %v", updated, err)
}
if updated.Repository.Url != "https://github.com/gopeed-test/repo" {
t.Fatal("mirror URL replaced canonical repository")
}
if official.Load() != 3 || badMirror.Load() != 3 {
t.Fatalf("attempts official=%d mirror=%d", official.Load(), badMirror.Load())
}
}
func TestExtensionGitShutdownWaitsForHandler(t *testing.T) {
d, _, _, _ := newExtensionGitFixture(t)
started := make(chan string, 1)
release := make(chan struct{})
t.Cleanup(func() {
select {
case <-release:
default:
close(release)
}
})
finished := make(chan error, 1)
go func() {
_, err := d.fetchExtensionByGit("https://github.com/gopeed-test/repo#nested/extension", func(path string, _ bool) (*Extension, error) {
started <- path
<-release
return d.parseExtensionByPath(path)
})
finished <- err
}()
var path string
select {
case path = <-started:
case err := <-finished:
t.Fatalf("Git operation finished before handler: %v", err)
case <-time.After(5 * time.Second):
t.Fatal("Git handler did not start")
}
closed := make(chan error, 1)
go func() { closed <- d.Close() }()
select {
case <-d.backgroundCtx.Done():
case <-time.After(5 * time.Second):
t.Fatal("shutdown did not cancel Git context")
}
select {
case err := <-closed:
t.Fatalf("shutdown returned before handler finished: %v", err)
case <-time.After(100 * time.Millisecond):
}
if _, err := os.Stat(filepath.Join(path, "manifest.json")); err != nil {
t.Fatalf("shutdown removed files still used by handler: %v", err)
}
if _, err := d.InstallExtensionByGit("https://github.com/gopeed-test/repo"); err == nil || !strings.Contains(err.Error(), "downloader is closed") {
t.Fatalf("expected new Git operations to be rejected during shutdown: %v", err)
}
close(release)
select {
case err := <-finished:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("Git handler did not finish")
}
select {
case err := <-closed:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("shutdown did not finish after Git handler")
}
}
+62
View File
@@ -0,0 +1,62 @@
package download
import (
"context"
"net"
"net/http"
"time"
gitclient "github.com/go-git/go-git/v5/plumbing/transport/client"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
)
func init() {
// go-git exposes a process-wide protocol registry. Register once during
// initialization, rather than changing it during concurrent clones. Proxy
// settings still come from each clone's ProxyOptions.
transport := githttp.NewClient(newExtensionGitHTTPClient(10*time.Second, 60*time.Second))
gitclient.InstallProtocol("http", transport)
gitclient.InstallProtocol("https", transport)
}
func newExtensionGitHTTPClient(connectTimeout, idleTimeout time.Duration) *http.Client {
dialer := &net.Dialer{Timeout: connectTimeout, KeepAlive: 30 * time.Second}
// A socket read deadline must apply to one transfer, rather than a shared
// HTTP/2 connection carrying traffic from other requests.
protocols := new(http.Protocols)
protocols.SetHTTP1(true)
return &http.Client{Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
conn, err := dialer.DialContext(ctx, network, address)
if err != nil {
return nil, err
}
return &gitIdleConn{Conn: conn, timeout: idleTimeout}, nil
},
TLSHandshakeTimeout: connectTimeout,
IdleConnTimeout: idleTimeout,
Protocols: protocols,
}}
}
type gitIdleConn struct {
net.Conn
timeout time.Duration
}
func (c *gitIdleConn) Read(p []byte) (int, error) {
if err := c.Conn.SetReadDeadline(time.Now().Add(c.timeout)); err != nil {
return 0, err
}
return c.Conn.Read(p)
}
// Refresh an already-pending read when a pooled HTTP/1 connection starts the
// next request; time spent idle in the pool must not consume its read allowance.
func (c *gitIdleConn) Write(p []byte) (int, error) {
if err := c.Conn.SetReadDeadline(time.Now().Add(c.timeout)); err != nil {
return 0, err
}
return c.Conn.Write(p)
}
@@ -0,0 +1,189 @@
package download
import (
"context"
"errors"
"io"
"net"
"net/http"
"net/http/httptest"
"net/http/httptrace"
"net/url"
"reflect"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/githubmirror"
)
func TestExtensionGitHTTPTransportAllowsSlowStreaming(t *testing.T) {
const chunks = 24
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
for range chunks {
if _, err := io.WriteString(w, "x"); err != nil {
return
}
w.(http.Flusher).Flush()
select {
case <-time.After(50 * time.Millisecond):
case <-r.Context().Done():
return
}
}
}))
defer server.Close()
client := newExtensionGitHTTPClient(time.Second, time.Second)
defer client.CloseIdleConnections()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
response, err := client.Do(request)
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
// A regularly streaming response lasts longer than the idle allowance.
body, err := io.ReadAll(response.Body)
if err != nil || len(body) != chunks {
t.Fatalf("received %d bytes, error=%v", len(body), err)
}
}
func TestExtensionGitHTTPTransportTimesOutStalledReads(t *testing.T) {
for _, bodyStarted := range []bool{false, true} {
name := "headers"
if bodyStarted {
name = "body"
}
t.Run(name, func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if bodyStarted {
_, _ = io.WriteString(w, "partial")
w.(http.Flusher).Flush()
}
<-r.Context().Done()
}))
defer server.Close()
client := newExtensionGitHTTPClient(time.Second, 100*time.Millisecond)
defer client.CloseIdleConnections()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
response, err := client.Do(request)
if err == nil {
defer response.Body.Close()
_, err = io.ReadAll(response.Body)
}
var timeout net.Error
if !errors.As(err, &timeout) || !timeout.Timeout() || ctx.Err() != nil {
t.Fatalf("expected idle timeout before parent deadline: error=%v context=%v", err, ctx.Err())
}
})
}
}
func TestExtensionGitIdleTimeoutFallsBackThroughProxy(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Hostname() == "github.com" {
_, _ = io.WriteString(w, "partial")
w.(http.Flusher).Flush()
<-r.Context().Done()
return
}
_, _ = io.WriteString(w, "complete")
}))
defer server.Close()
client := newExtensionGitHTTPClient(time.Second, 100*time.Millisecond)
defer client.CloseIdleConnections()
proxy, _ := url.Parse(server.URL)
client.Transport.(*http.Transport).Proxy = http.ProxyURL(proxy)
config := &githubmirror.Config{Enabled: true, Mirrors: []githubmirror.Mirror{{Type: "ghProxy", URL: server.URL}}}
source := "http://github.com/test/repo.git"
var tried []string
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
result, err := githubmirror.Fallback(ctx, config, source, githubmirror.Options[string]{Kind: githubmirror.Repository}, func(ctx context.Context, candidate string) (string, error) {
tried = append(tried, candidate)
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, candidate, nil)
response, err := client.Do(request)
if err != nil {
return "", err
}
defer response.Body.Close()
body, err := io.ReadAll(response.Body)
return string(body), err
})
if err != nil || result != "complete" || !reflect.DeepEqual(tried, []string{source, server.URL + "/" + source}) {
t.Fatalf("result=%q tried=%v error=%v", result, tried, err)
}
}
func TestExtensionGitHTTPTransportCancelsStalledBody(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.(http.Flusher).Flush()
<-r.Context().Done()
}))
defer server.Close()
client := newExtensionGitHTTPClient(time.Second, time.Minute)
defer client.CloseIdleConnections()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
response, err := client.Do(request)
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
cancel()
done := make(chan error, 1)
go func() { _, err := io.ReadAll(response.Body); done <- err }()
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Fatalf("error=%v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("parent cancellation did not interrupt the body")
}
}
func TestExtensionGitHTTPTransportRefreshesReusedConnection(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/slow" {
select {
case <-time.After(500 * time.Millisecond):
case <-r.Context().Done():
return
}
}
_, _ = io.WriteString(w, "ok")
}))
defer server.Close()
client := newExtensionGitHTTPClient(time.Second, time.Second)
defer client.CloseIdleConnections()
response, err := client.Get(server.URL)
if err != nil {
t.Fatal(err)
}
_, err = io.Copy(io.Discard, response.Body)
_ = response.Body.Close()
if err != nil {
t.Fatal(err)
}
time.Sleep(700 * time.Millisecond)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
reused := false
ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{GotConn: func(info httptrace.GotConnInfo) { reused = info.Reused }})
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL+"/slow", nil)
response, err = client.Do(request)
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
body, err := io.ReadAll(response.Body)
if err != nil || string(body) != "ok" || !reused {
t.Fatalf("reused=%v body=%q error=%v", reused, body, err)
}
}
+3 -42
View File
@@ -1070,54 +1070,15 @@ func TestDownloader_Extension_BlobRecoverOnError(t *testing.T) {
} }
func TestDownloader_InstallExtensionByGit(t *testing.T) { func TestDownloader_InstallExtensionByGit(t *testing.T) {
setupDownloader(func(downloader *Downloader) { testInstallExtensionGitURL(t, "https://github.com/gopeed-test/repo#nested/extension")
if _, err := downloader.InstallExtensionByGit("https://github.com/GopeedLab/gopeed-extension-samples#github-release-sample"); err != nil {
t.Fatal(err)
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases",
})
if err != nil {
t.Fatal(err)
}
if len(rr.Res.Files) == 1 {
t.Fatal("resolve error")
}
})
} }
func TestDownloader_InstallExtensionByGitSimple(t *testing.T) { func TestDownloader_InstallExtensionByGitSimple(t *testing.T) {
setupDownloader(func(downloader *Downloader) { testInstallExtensionGitURL(t, "github.com/gopeed-test/repo#nested/extension")
if _, err := downloader.InstallExtensionByGit("github.com/GopeedLab/gopeed-extension-samples#github-release-sample"); err != nil {
t.Fatal(err)
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases",
})
if err != nil {
t.Fatal(err)
}
if len(rr.Res.Files) == 1 {
t.Fatal("resolve error")
}
})
} }
func TestDownloader_InstallExtensionByGitFull(t *testing.T) { func TestDownloader_InstallExtensionByGitFull(t *testing.T) {
setupDownloader(func(downloader *Downloader) { testInstallExtensionGitURL(t, "https://github.com/gopeed-test/repo.git#nested/extension")
if _, err := downloader.InstallExtensionByGit("https://github.com/GopeedLab/gopeed-extension-samples.git#github-release-sample"); err != nil {
t.Fatal(err)
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases",
})
if err != nil {
t.Fatal(err)
}
if len(rr.Res.Files) == 1 {
t.Fatal("resolve error")
}
})
} }
func TestDownloader_UpgradeExtension(t *testing.T) { func TestDownloader_UpgradeExtension(t *testing.T) {
+151
View File
@@ -0,0 +1,151 @@
package download
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
"time"
"github.com/GopeedLab/gopeed/internal/githubmirror"
"github.com/GopeedLab/gopeed/internal/httpclient"
)
func (d *Downloader) resourceClient(publicOnly bool) (*http.Client, error) {
cfg, err := d.GetConfig()
if err != nil {
return nil, err
}
options := httpclient.Options{
Client: httpclient.ClientOptions{Timeout: 30 * time.Second},
Transport: httpclient.TransportOptions{Proxy: cfg.Proxy.ToHandler(), TLSHandshakeTimeout: 10 * time.Second},
}
if publicOnly {
// A configured upstream proxy is trusted, but target URLs are still checked.
options.Client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 5 {
return errors.New("too many redirects")
}
return validatePublicURL(req.Context(), req.URL.String())
}
if cfg.Proxy == nil || !cfg.Proxy.Enable {
options.Transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
host, port, err := net.SplitHostPort(address)
if err != nil {
return nil, err
}
ips, err := publicIPs(ctx, host)
if err != nil {
return nil, err
}
var last error
for _, ip := range ips {
conn, err := (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
if err == nil {
return conn, nil
}
last = err
}
return nil, last
}
}
}
return httpclient.NewClient(options)
}
func publicIPs(ctx context.Context, host string) ([]net.IPAddr, error) {
ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, err
}
if len(ips) == 0 {
return nil, errors.New("target has no addresses")
}
for _, ip := range ips {
if !ip.IP.IsGlobalUnicast() || ip.IP.IsPrivate() || ip.IP.IsLoopback() || ip.IP.IsLinkLocalUnicast() || ip.IP.IsUnspecified() {
return nil, errors.New("private or local resource target is not allowed")
}
}
return ips, nil
}
func validatePublicURL(ctx context.Context, raw string) error {
u, err := url.Parse(raw)
if err != nil || u == nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" || u.User != nil {
return errors.New("invalid resource URL")
}
_, err = publicIPs(ctx, u.Hostname())
return err
}
func (d *Downloader) fetchPublicResource(ctx context.Context, raw string, limit int64, validate func([]byte) error) ([]byte, error) {
return d.fetchResource(ctx, raw, limit, true, validate)
}
// fetchResource races complete, validated responses. A successful HEAD alone
// cannot win, and failed candidates never discard a usable official response.
func (d *Downloader) fetchResource(ctx context.Context, raw string, limit int64, publicOnly bool, validate func([]byte) error) ([]byte, error) {
cfg, err := d.GetConfig()
if err != nil {
return nil, err
}
client, err := d.resourceClient(publicOnly)
if err != nil {
return nil, err
}
defer client.CloseIdleConnections()
return githubmirror.Race(ctx, cfg.GithubMirror, raw,
githubmirror.Options[[]byte]{Kind: githubmirror.Resource, Timeout: 30 * time.Second},
func(ctx context.Context, candidate string) ([]byte, error) {
if publicOnly {
if err := validatePublicURL(ctx, candidate); err != nil {
return nil, err
}
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, candidate, nil)
if err != nil {
return nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("resource returned HTTP %d", response.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(response.Body, limit+1))
if err != nil {
return nil, err
}
if int64(len(body)) > limit {
return nil, errors.New("resource exceeds size limit")
}
if len(body) == 0 {
return nil, errors.New("empty resource")
}
if validate != nil {
if err := validate(body); err != nil {
return nil, err
}
}
return body, nil
},
)
}
// githubResourceCandidates accepts official GitHub HTTPS resource URLs only.
func githubResourceCandidates(config *githubmirror.Config, raw string) ([]string, error) {
u, err := url.Parse(raw)
if err != nil || u == nil || u.Scheme != "https" || u.User != nil || u.Port() != "" || u.Fragment != "" || strings.Contains(raw, "#") {
return nil, ErrInvalidMirrorURL
}
u.Host = strings.ToLower(u.Host)
switch u.Host {
case "github.com", "raw.githubusercontent.com":
default:
return nil, ErrInvalidMirrorURL
}
return githubmirror.Candidates(config, u.String(), githubmirror.Resource), nil
}
+35
View File
@@ -0,0 +1,35 @@
package download
import (
"errors"
"github.com/GopeedLab/gopeed/pkg/base"
)
var (
ErrMirrorProviderNotFound = errors.New("mirror provider not found")
ErrInvalidMirrorURL = errors.New("invalid mirror resource URL")
)
// Each provider owns URL validation and candidate generation using its own
// configuration. Providers only generate URLs; they must not fetch resources.
type mirrorCandidateProvider func(config *base.DownloaderStoreConfig, raw string) ([]string, error)
// Add providers here without changing the public API or its client transport.
var mirrorCandidateProviders = map[string]mirrorCandidateProvider{
"github": func(config *base.DownloaderStoreConfig, raw string) ([]string, error) {
return githubResourceCandidates(config.GithubMirror, raw)
},
}
func (d *Downloader) GetMirrorCandidates(provider, raw string) ([]string, error) {
candidates, ok := mirrorCandidateProviders[provider]
if !ok {
return nil, ErrMirrorProviderNotFound
}
config, err := d.GetConfig()
if err != nil {
return nil, err
}
return candidates(config, raw)
}
+172
View File
@@ -0,0 +1,172 @@
package download
import (
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/util"
)
func (d *Downloader) protocolManager(name string) (fetcher.FetcherManager, error) {
for _, fm := range d.cfg.FetchManagers {
if fm.Name() == name {
return fm, nil
}
}
return nil, fmt.Errorf("%w: %s", ErrUnSupportedProtocol, name)
}
func jsonObject(v any) (map[string]json.RawMessage, error) {
raw, err := json.Marshal(v)
if err != nil {
return nil, err
}
var object map[string]json.RawMessage
if err = json.Unmarshal(raw, &object); err != nil {
return nil, err
}
if object == nil {
object = map[string]json.RawMessage{}
}
return object, nil
}
func normalizeProtocol(fm fetcher.FetcherManager, v any) (any, error) {
raw, err := json.Marshal(v)
if err != nil {
return nil, err
}
target := fm.DefaultConfig()
if target == nil {
return nil, errors.New("protocol does not expose configuration")
}
// Managers may return either typed pointers or value/map defaults.
valueDefault := reflect.TypeOf(target).Kind() != reflect.Pointer
if valueDefault {
pointer := reflect.New(reflect.TypeOf(target))
defaults, err := json.Marshal(target)
if err != nil {
return nil, err
}
if err = json.Unmarshal(defaults, pointer.Interface()); err != nil {
return nil, err
}
target = pointer.Interface()
}
if err = json.Unmarshal(raw, target); err != nil {
return nil, err
}
if normalizer, ok := fm.(fetcher.ConfigNormalizer); ok {
raw, err = json.Marshal(target)
if err != nil {
return nil, err
}
return normalizer.NormalizeConfig(raw)
}
if valueDefault {
return reflect.ValueOf(target).Elem().Interface(), nil
}
return target, nil
}
// preserveProtocolState ignores all client-supplied state, including null or
// malformed state values, before decoding/validating the writable config.
func preserveProtocolState(next, previous any) (any, error) {
object, err := jsonObject(next)
if err != nil {
return nil, err
}
before, err := jsonObject(previous)
if err != nil {
return nil, err
}
// Match encoding/json field folding as well as the canonical spelling.
for key := range object {
if strings.EqualFold(key, "state") {
delete(object, key)
}
}
if state, ok := before["state"]; ok {
object["state"] = state
}
return object, nil
}
func (d *Downloader) readProtocolConfig(name string) (any, error) {
if _, err := d.protocolManager(name); err != nil {
return nil, err
}
d.configLock.RLock()
defer d.configLock.RUnlock()
value := d.cfg.ProtocolConfig[name]
return *util.DeepClone(&value), nil
}
// updateProtocolState is the only internal write path for backend-owned state.
// It never publishes callback edits outside the reserved state section.
func (d *Downloader) updateProtocolState(name string, change func(config, state json.RawMessage) (any, error)) error {
fm, err := d.protocolManager(name)
if err != nil {
return err
}
d.configLock.Lock()
defer d.configLock.Unlock()
if d.closed.Load() {
return errors.New("downloader is closed")
}
next := util.DeepClone(d.cfg.DownloaderStoreConfig)
object, err := jsonObject(next.ProtocolConfig[name])
if err != nil {
return err
}
state, ok := object["state"]
if !ok {
return fmt.Errorf("protocol %s does not expose persistent state", name)
}
raw, err := json.Marshal(object)
if err != nil {
return err
}
value, err := change(raw, state)
if err != nil {
return err
}
// Keep the transaction's original config isolated from callback mutations.
object["state"], err = json.Marshal(value)
if err != nil {
return err
}
normalized, err := normalizeProtocol(fm, object)
if err != nil {
return err
}
next.ProtocolConfig[name] = normalized
if err = d.storage.Put(bucketConfig, "config", next); err != nil {
return err
}
d.cfg.DownloaderStoreConfig.ProtocolConfig = next.ProtocolConfig
return nil
}
// initProtocolConfigs loads the current schema and initializes missing defaults.
// This release intentionally does not import settings from older config layouts.
func (d *Downloader) initProtocolConfigs() error {
cfg := d.cfg.DownloaderStoreConfig
cfg.Init()
if err := cfg.GithubMirror.Validate(); err != nil {
return err
}
for _, fm := range d.cfg.FetchManagers {
value, ok := cfg.ProtocolConfig[fm.Name()]
if !ok {
value = map[string]any{}
}
normalized, err := normalizeProtocol(fm, value)
if err != nil {
return fmt.Errorf("%s config: %w", fm.Name(), err)
}
cfg.ProtocolConfig[fm.Name()] = normalized
}
return d.storage.Put(bucketConfig, "config", cfg)
}
+250
View File
@@ -0,0 +1,250 @@
package download
import (
"context"
"encoding/json"
"errors"
"fmt"
"regexp"
"sync"
"sync/atomic"
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/util"
)
var ErrProtocolJobNotFound = errors.New("protocol job not found")
var jobNamePattern = regexp.MustCompile(`^[a-z][a-z0-9-]*$`)
type protocolJobState struct {
LastSuccess time.Time `json:"lastSuccess"`
Failures int `json:"failures"`
}
type protocolJobRun struct {
done chan struct{}
err error
source fetcher.TriggerSource
}
type registeredJob struct {
protocol string
spec fetcher.CronJob
mu sync.Mutex
state protocolJobState
active *protocolJobRun
next time.Time
wake chan struct{}
force atomic.Bool
}
type protocolScheduler struct {
d *Downloader
jobs map[string]*registeredJob
wg sync.WaitGroup
// gate prevents adding work while shutdown waits for it.
gate sync.Mutex
stopped bool
}
func (d *Downloader) setupProtocolJobs() error {
scheduler := &protocolScheduler{d: d, jobs: map[string]*registeredJob{}}
for _, fm := range d.cfg.FetchManagers {
provider, ok := fm.(fetcher.CronProvider)
if !ok {
continue
}
name := fm.Name()
host := fetcher.ProtocolHost{
Read: func(dst any) error {
value, err := d.readProtocolConfig(name)
if err != nil {
return err
}
return util.MapToStruct(value, dst)
},
UpdateState: func(change func(config, state json.RawMessage) (any, error)) error {
return d.updateProtocolState(name, change)
},
Fetch: func(ctx context.Context, raw string, limit int64, validate func([]byte) error) ([]byte, error) {
return d.fetchPublicResource(ctx, raw, limit, validate)
},
}
for _, spec := range provider.CronJobs(host) {
if !jobNamePattern.MatchString(spec.Name) || spec.Every <= 0 || spec.Timeout <= 0 || spec.Run == nil {
return fmt.Errorf("invalid protocol job: %s/%s", name, spec.Name)
}
key := name + "/" + spec.Name
if _, ok := scheduler.jobs[key]; ok {
return fmt.Errorf("duplicate protocol job: %s", key)
}
entry := &registeredJob{protocol: name, spec: spec, wake: make(chan struct{}, 1)}
if _, err := d.storage.Get(bucketProtocolJobs, key, &entry.state); err != nil {
return err
}
scheduler.jobs[key] = entry
}
}
d.protocolJobs = scheduler
return nil
}
func (s *protocolScheduler) start() {
for key, entry := range s.jobs {
s.wg.Add(1)
go s.loop(key, entry)
}
}
func (s *protocolScheduler) signal(entry *registeredJob) {
select {
case entry.wake <- struct{}{}:
default:
}
}
func (s *protocolScheduler) changed(protocol string) {
for _, entry := range s.jobs {
if protocol == "" || entry.protocol == protocol {
entry.force.Store(true)
s.signal(entry)
}
}
}
func (s *protocolScheduler) loop(key string, entry *registeredJob) {
defer s.wg.Done()
entry.mu.Lock()
if entry.next.IsZero() {
entry.next = entry.state.LastSuccess.Add(entry.spec.Every)
if entry.next.Before(time.Now()) {
entry.next = time.Now().Add(time.Second)
}
}
next := entry.next
entry.mu.Unlock()
timer := time.NewTimer(time.Until(next))
defer timer.Stop()
for {
select {
case <-s.d.backgroundCtx.Done():
return
case <-entry.wake:
entry.mu.Lock()
next = entry.next
entry.mu.Unlock()
if entry.force.Swap(false) {
next = time.Now()
}
case <-timer.C:
_ = s.trigger(s.d.backgroundCtx, key, fetcher.TriggerScheduled)
entry.mu.Lock()
next = entry.next
entry.mu.Unlock()
}
if !timer.Stop() {
select {
case <-timer.C:
default:
}
}
timer.Reset(max(time.Until(next), time.Millisecond))
}
}
func (s *protocolScheduler) trigger(ctx context.Context, key string, source fetcher.TriggerSource) error {
entry, ok := s.jobs[key]
if !ok {
return fmt.Errorf("%w: %s", ErrProtocolJobNotFound, key)
}
for {
if err := ctx.Err(); err != nil {
return err
}
s.gate.Lock()
if s.stopped || s.d.closed.Load() {
s.gate.Unlock()
return errors.New("downloader is closed")
}
entry.mu.Lock()
run := entry.active
if run == nil {
run = &protocolJobRun{done: make(chan struct{}), source: source}
entry.active = run
s.wg.Add(1)
go s.execute(key, entry, run, source)
}
entry.mu.Unlock()
s.gate.Unlock()
select {
case <-ctx.Done():
return ctx.Err()
case <-run.done:
// A manual request must still run when the scheduled attempt it
// joined was skipped (for example, automatic refresh is disabled).
if source == fetcher.TriggerManual && run.source == fetcher.TriggerScheduled && errors.Is(run.err, fetcher.ErrJobSkipped) {
continue
}
return run.err
}
}
}
func (s *protocolScheduler) execute(key string, entry *registeredJob, run *protocolJobRun, source fetcher.TriggerSource) {
defer s.wg.Done()
ctx, cancel := context.WithTimeout(s.d.backgroundCtx, entry.spec.Timeout)
defer cancel()
err := func() (err error) {
defer func() {
if v := recover(); v != nil {
err = fmt.Errorf("protocol job panic: %v", v)
}
}()
return entry.spec.Run(ctx, source)
}()
if err == nil {
err = ctx.Err()
}
entry.mu.Lock()
switch {
case err == nil:
entry.state.LastSuccess = time.Now().UTC()
entry.state.Failures = 0
case errors.Is(err, fetcher.ErrJobSkipped), errors.Is(err, fetcher.ErrConfigChanged), errors.Is(err, context.Canceled):
default:
entry.state.Failures++
}
if storeErr := s.d.storage.Put(bucketProtocolJobs, key, entry.state); storeErr != nil {
err = errors.Join(err, storeErr)
}
switch {
case err == nil:
entry.next = entry.state.LastSuccess.Add(entry.spec.Every)
case errors.Is(err, fetcher.ErrJobSkipped):
entry.next = time.Now().Add(entry.spec.Every)
case errors.Is(err, fetcher.ErrConfigChanged):
entry.next = time.Now().Add(time.Second)
default:
retry := time.Minute * time.Duration(1<<min(max(entry.state.Failures-1, 0), 6))
entry.next = time.Now().Add(retry)
}
run.err = err
entry.active = nil
close(run.done)
entry.mu.Unlock()
if err != nil && !errors.Is(err, fetcher.ErrJobSkipped) && !errors.Is(err, context.Canceled) {
s.d.Logger.Warn().Err(err).Str("job", key).Str("source", string(source)).Msg("protocol job failed")
}
if source == fetcher.TriggerManual {
s.signal(entry)
}
}
func (s *protocolScheduler) close() {
s.gate.Lock()
s.stopped = true
s.gate.Unlock()
s.wg.Wait()
}
func (d *Downloader) TriggerProtocolJob(ctx context.Context, protocol, name string) error {
if d.protocolJobs == nil {
return ErrProtocolJobNotFound
}
err := d.protocolJobs.trigger(ctx, protocol+"/"+name, fetcher.TriggerManual)
if errors.Is(err, fetcher.ErrJobSkipped) {
return nil
}
return err
}
+89
View File
@@ -0,0 +1,89 @@
package download
import (
"context"
"errors"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
)
type jobWaitContext struct {
context.Context
waiting chan struct{}
once sync.Once
}
func (c *jobWaitContext) Done() <-chan struct{} {
c.once.Do(func() { close(c.waiting) })
return c.Context.Done()
}
func TestManualProtocolJobRunsAfterJoinedScheduledSkip(t *testing.T) {
entered := make(chan struct{})
release := make(chan struct{})
var manualCalls atomic.Int32
manager := &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Second,
Run: func(ctx context.Context, source fetcher.TriggerSource) error {
if source == fetcher.TriggerManual {
manualCalls.Add(1)
return nil
}
close(entered)
select {
case <-release:
return fetcher.ErrJobSkipped
case <-ctx.Done():
return ctx.Err()
}
},
}}
d := testBackendDownloader(t, manager)
scheduled := make(chan error, 1)
go func() {
scheduled <- d.protocolJobs.trigger(context.Background(), "test/refresh", fetcher.TriggerScheduled)
}()
<-entered
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
waiter := &jobWaitContext{Context: ctx, waiting: make(chan struct{})}
manual := make(chan error, 1)
go func() { manual <- d.TriggerProtocolJob(waiter, "test", "refresh") }()
<-waiter.waiting
close(release)
if err := <-scheduled; !errors.Is(err, fetcher.ErrJobSkipped) {
t.Fatal(err)
}
if err := <-manual; err != nil {
t.Fatal(err)
}
if manualCalls.Load() != 1 {
t.Fatal("manual refresh was swallowed by skipped automatic work")
}
}
func TestManualProtocolJobFailureKeepsScheduledBackoff(t *testing.T) {
var calls atomic.Int32
failure := errors.New("offline")
manager := &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Second,
Run: func(context.Context, fetcher.TriggerSource) error { calls.Add(1); return failure },
}}
d := testBackendDownloader(t, manager)
if err := d.TriggerProtocolJob(context.Background(), "test", "refresh"); !errors.Is(err, failure) {
t.Fatal(err)
}
entry := d.protocolJobs.jobs["test/refresh"]
entry.mu.Lock()
next := entry.next
entry.mu.Unlock()
if time.Until(next) < 59*time.Second {
t.Fatal("manual failure did not schedule retry backoff")
}
time.Sleep(50 * time.Millisecond)
if calls.Load() != 1 {
t.Fatalf("manual failure immediately retried %d times", calls.Load())
}
}
+340
View File
@@ -0,0 +1,340 @@
package download
import (
"encoding/json"
"errors"
"reflect"
"sync"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
btProtocol "github.com/GopeedLab/gopeed/internal/protocol/bt"
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
"github.com/GopeedLab/gopeed/pkg/base"
)
// State tests initialize config without starting network clients or cron loops.
func protocolConfigDownloader(t *testing.T, storage Storage, managers ...fetcher.FetcherManager) *Downloader {
t.Helper()
if storage == nil {
storage = NewMemStorage()
}
d := NewDownloader(&DownloaderConfig{Storage: storage, StorageDir: t.TempDir(), FetchManagers: managers})
d.cfg.DownloaderStoreConfig = &base.DownloaderStoreConfig{}
if err := storage.Setup([]string{bucketConfig}); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { d.cancelBackground(); _ = storage.Close() })
if err := d.initProtocolConfigs(); err != nil {
t.Fatal(err)
}
return d
}
func protocolObject(t *testing.T, d *Downloader, name string) map[string]any {
t.Helper()
cfg, err := d.GetConfig()
if err != nil {
t.Fatal(err)
}
raw, err := json.Marshal(cfg.ProtocolConfig[name])
if err != nil {
t.Fatal(err)
}
var result map[string]any
if err := json.Unmarshal(raw, &result); err != nil {
t.Fatal(err)
}
return result
}
func configureBT(t *testing.T, d *Downloader, urls ...string) {
t.Helper()
cfg, _ := d.GetConfig()
cfg.ProtocolConfig["bt"] = map[string]any{"autoUpdateTrackers": false, "trackerSubscribeUrls": urls, "customTrackers": []string{"udp://custom.example:80/announce"}}
if err := d.PutConfig(cfg); err != nil {
t.Fatal(err)
}
}
func seedBTState(t *testing.T, d *Downloader) {
t.Helper()
if err := d.updateProtocolState("bt", func(_, _ json.RawMessage) (any, error) {
return map[string]any{
"trackerSources": map[string][]string{"https://example.com/list": {"udp://cached.example:80/announce"}},
"trackerSubscriptionStatuses": map[string]any{"https://example.com/list": map[string]any{"success": true, "trackerCount": 1, "lastFetchTime": time.Date(2026, 10, 19, 1, 2, 3, 0, time.UTC)}},
"lastTrackerUpdateTime": time.Date(2026, 10, 19, 1, 2, 3, 0, time.UTC),
}, nil
}); err != nil {
t.Fatal(err)
}
}
func TestFullConfigValidatesProtocolsAndPreservesExplicitValues(t *testing.T) {
d := protocolConfigDownloader(t, nil, &httpProtocol.FetcherManager{}, &btProtocol.FetcherManager{})
configureBT(t, d)
cfg, _ := d.GetConfig()
cfg.ProtocolConfig["http"] = map[string]any{"connections": 0, "useServerCtime": false}
if err := d.PutConfig(cfg); err != nil {
t.Fatal(err)
}
got := protocolObject(t, d, "http")
if got["connections"] != float64(0) || got["useServerCtime"] != false {
t.Fatal(got)
}
for _, invalid := range []any{map[string]any{"connections": "bad"}, map[string]any{"useServerCtime": "bad"}} {
cfg, _ := d.GetConfig()
cfg.ProtocolConfig["http"] = invalid
if err := d.PutConfig(cfg); err == nil {
t.Fatalf("invalid protocol config accepted: %v", invalid)
}
}
cfg, _ = d.GetConfig()
cfg.DownloadDir = "must-not-publish"
cfg.ProtocolConfig["bt"] = map[string]any{"listenPort": -1}
if err := d.PutConfig(cfg); err == nil {
t.Fatal("invalid BT config accepted")
}
saved, _ := d.GetConfig()
if saved.DownloadDir == "must-not-publish" {
t.Fatal("invalid full PUT partially published")
}
cfg, _ = d.GetConfig()
cfg.ProtocolConfig["unknown"] = map[string]any{"state": map[string]any{"injected": true}}
if err := d.PutConfig(cfg); err == nil {
t.Fatal("unregistered protocol accepted")
}
// Returned configs are detached snapshots.
snapshot, _ := d.GetConfig()
snapshot.ProtocolConfig["http"] = map[string]any{"connections": 999}
if !reflect.DeepEqual(protocolObject(t, d, "http"), got) {
t.Fatal("snapshot changed live config")
}
}
func TestProtocolConfigIgnoresUnknownFields(t *testing.T) {
for _, mode := range []string{"stored", "put"} {
t.Run(mode, func(t *testing.T) {
d := protocolConfigDownloader(t, nil, &btProtocol.FetcherManager{})
cfg, _ := d.GetConfig()
cfg.ProtocolConfig["bt"] = map[string]any{
"listenPort": 6881,
"autoUpdateTrackers": false,
"trackers": []string{"udp://obsolete.example:80/announce"},
"state": map[string]any{"obsolete": true},
}
if mode == "stored" {
if err := d.storage.Put(bucketConfig, "config", cfg); err != nil {
t.Fatal(err)
}
var restored base.DownloaderStoreConfig
if exists, err := d.storage.Get(bucketConfig, "config", &restored); err != nil || !exists {
t.Fatalf("load stored config: %v %v", exists, err)
}
d.cfg.DownloaderStoreConfig = &restored
if err := d.initProtocolConfigs(); err != nil {
t.Fatalf("load config containing removed fields: %v", err)
}
} else if err := d.PutConfig(cfg); err != nil {
t.Fatalf("save config containing unknown fields: %v", err)
}
actual := protocolObject(t, d, "bt")
if actual["listenPort"] != float64(6881) || actual["autoUpdateTrackers"] != false {
t.Fatalf("known fields changed: %v", actual)
}
if _, exists := actual["trackers"]; exists {
t.Fatal("removed field retained")
}
if len(actual["customTrackers"].([]any)) != 0 {
t.Fatal("removed trackers migrated into custom trackers")
}
if _, exists := actual["state"].(map[string]any)["obsolete"]; exists {
t.Fatal("unknown state field retained")
}
})
}
}
func TestBTStateSurvivesOmittedNullForgedAndStaleFullConfig(t *testing.T) {
for _, mode := range []string{"omitted", "null", "forged", "wrong-type", "stale", "omitted-protocol", "omitted-timestamp", "case-variant", "unicode-variant"} {
t.Run(mode, func(t *testing.T) {
d := protocolConfigDownloader(t, nil, &btProtocol.FetcherManager{})
configureBT(t, d, "https://example.com/list")
stale, _ := d.GetConfig()
seedBTState(t, d)
expected := protocolObject(t, d, "bt")["state"]
cfg, _ := d.GetConfig()
bt := protocolObject(t, d, "bt")
switch mode {
case "omitted":
delete(bt, "state")
case "null":
bt["state"] = nil
case "forged":
bt["state"] = map[string]any{"lastTrackerUpdateTime": "2000-01-01T00:00:00Z", "trackerSources": map[string]any{}, "unknown": true}
case "wrong-type":
bt["state"] = "ignore this before decoding"
case "stale":
cfg = stale
bt = map[string]any{"autoUpdateTrackers": false, "trackerSubscribeUrls": []string{"https://example.com/list"}, "customTrackers": []string{"udp://custom.example:80/announce"}, "state": map[string]any{}}
case "omitted-protocol":
delete(cfg.ProtocolConfig, "bt")
case "case-variant":
bt["STATE"] = "ignore this alias"
case "unicode-variant":
bt["ſtate"] = "ignore Unicode case folding"
case "omitted-timestamp":
bt["state"] = map[string]any{"trackerSources": map[string]any{}}
}
if mode != "omitted-protocol" {
bt["listenPort"] = 6881
cfg.ProtocolConfig["bt"] = bt
}
if err := d.PutConfig(cfg); err != nil {
t.Fatal(err)
}
actual := protocolObject(t, d, "bt")
if !reflect.DeepEqual(actual["state"], expected) {
t.Fatalf("backend state replaced: %v", actual)
}
if _, exists := actual["trackers"]; exists {
t.Fatal("derived tracker list persisted")
}
if mode != "omitted-protocol" && actual["listenPort"] != float64(6881) {
t.Fatal("writable config did not change")
}
})
}
}
func TestBTConfigRemovalPrunesOnlyRemovedState(t *testing.T) {
d := protocolConfigDownloader(t, nil, &btProtocol.FetcherManager{})
configureBT(t, d, "https://example.com/list")
seedBTState(t, d)
configureBT(t, d, "https://new.example/list")
state := protocolObject(t, d, "bt")["state"].(map[string]any)
if len(state["trackerSources"].(map[string]any)) != 0 || len(state["trackerSubscriptionStatuses"].(map[string]any)) != 0 {
t.Fatal("removed subscription retained state")
}
}
type counterState struct {
Count int `json:"count"`
Other int `json:"other"`
}
type counterConfig struct {
fetcher.ConfigState[counterState]
Writable int `json:"writable"`
}
type counterManager struct{ httpProtocol.FetcherManager }
func (*counterManager) Name() string { return "counter" }
func (*counterManager) DefaultConfig() any { return &counterConfig{Writable: 7} }
func counterHost(d *Downloader) fetcher.ProtocolHost {
return fetcher.ProtocolHost{UpdateState: func(change func(config, state json.RawMessage) (any, error)) error {
return d.updateProtocolState("counter", change)
}}
}
func TestProtocolUpdateStateCommitsOnlyStateAndUsesLatestConfig(t *testing.T) {
d := protocolConfigDownloader(t, nil, &counterManager{})
host := counterHost(d)
var wg sync.WaitGroup
for i := 0; i < 20; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if err := fetcher.UpdateState[counterConfig, counterState](host, func(cfg *counterConfig, state *counterState) error { cfg.Writable = 999; state.Count++; return nil }); err != nil {
t.Error(err)
}
}()
}
wg.Wait()
actual := protocolObject(t, d, "counter")
if actual["writable"] != float64(7) || actual["state"].(map[string]any)["count"] != float64(20) {
t.Fatal(actual)
}
// Rejecting a callback must not publish its partial mutation.
err := fetcher.UpdateState[counterConfig, counterState](host, func(_ *counterConfig, state *counterState) error { state.Count = 1000; return errors.New("reject") })
if err == nil || !reflect.DeepEqual(protocolObject(t, d, "counter"), actual) {
t.Fatal("failed callback published state")
}
if err := d.updateProtocolState("counter", func(_, _ json.RawMessage) (any, error) { return map[string]any{"count": "bad"}, nil }); err == nil {
t.Fatal("invalid state accepted")
}
}
func TestProtocolStateAndFullPutDoNotLoseConcurrentUpdates(t *testing.T) {
d := protocolConfigDownloader(t, nil, &counterManager{})
snapshot, _ := d.GetConfig()
snapshot.ProtocolConfig["counter"] = map[string]any{"writable": 21}
var wg sync.WaitGroup
for i := 0; i < 20; i++ {
wg.Add(2)
go func() {
defer wg.Done()
if err := d.PutConfig(snapshot); err != nil {
t.Error(err)
}
}()
go func() {
defer wg.Done()
if err := fetcher.UpdateState[counterConfig, counterState](counterHost(d), func(cfg *counterConfig, state *counterState) error { state.Count++; return nil }); err != nil {
t.Error(err)
}
}()
}
wg.Wait()
actual := protocolObject(t, d, "counter")
if actual["writable"] != float64(21) || actual["state"].(map[string]any)["count"] != float64(20) {
t.Fatal(actual)
}
}
func TestBTSubscriptionStatusesSurviveStorageRestart(t *testing.T) {
dir := t.TempDir()
storage := NewBoltStorage(dir)
d := protocolConfigDownloader(t, storage, &btProtocol.FetcherManager{})
configureBT(t, d, "https://example.com/list")
seedBTState(t, d)
expected := protocolObject(t, d, "bt")["state"]
if err := storage.Close(); err != nil {
t.Fatal(err)
}
reopened := NewBoltStorage(dir)
if err := reopened.Setup([]string{bucketConfig}); err != nil {
t.Fatal(err)
}
defer reopened.Close()
var persisted base.DownloaderStoreConfig
if exists, err := reopened.Get(bucketConfig, "config", &persisted); err != nil || !exists {
t.Fatalf("reload persisted config: %v %v", exists, err)
}
raw, _ := json.Marshal(persisted.ProtocolConfig["bt"])
var cfg map[string]any
if err := json.Unmarshal(raw, &cfg); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(cfg["state"], expected) {
t.Fatalf("state lost after reopening storage: %+v", cfg)
}
}
type rejectedConfigStorage struct{ Storage }
func (*rejectedConfigStorage) Put(string, string, any) error {
return errors.New("storage unavailable")
}
func TestProtocolStateAndPutStorageFailureDoNotPublish(t *testing.T) {
d := protocolConfigDownloader(t, nil, &counterManager{})
before := protocolObject(t, d, "counter")
d.storage = &rejectedConfigStorage{Storage: d.storage}
err := fetcher.UpdateState[counterConfig, counterState](counterHost(d), func(_ *counterConfig, state *counterState) error { state.Count = 99; return nil })
if err == nil || !reflect.DeepEqual(protocolObject(t, d, "counter"), before) {
t.Fatal("failed state persistence published a change")
}
cfg, _ := d.GetConfig()
cfg.ProtocolConfig["counter"] = map[string]any{"writable": 99}
if err := d.PutConfig(cfg); err == nil || !reflect.DeepEqual(protocolObject(t, d, "counter"), before) {
t.Fatal("failed config persistence published a change")
}
}
+12 -9
View File
@@ -191,13 +191,16 @@ func ensureScriptExecutable(t *testing.T, scriptPath string) {
} }
func setupScriptTest(t *testing.T, fn func(downloader *Downloader)) { func setupScriptTest(t *testing.T, fn func(downloader *Downloader)) {
defaultDownloader.Setup() t.Helper()
defaultDownloader.cfg.StorageDir = ".test_storage" // Each test owns its runtime; a closed Downloader cannot be reused.
defaultDownloader.cfg.DownloadDir = ".test_download" d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
defer func() { if err := d.Setup(); err != nil {
defaultDownloader.Clear() t.Fatal(err)
os.RemoveAll(defaultDownloader.cfg.StorageDir) }
os.RemoveAll(defaultDownloader.cfg.DownloadDir) t.Cleanup(func() {
}() if err := d.Close(); err != nil {
fn(defaultDownloader) t.Error(err)
}
})
fn(d)
} }
+12 -10
View File
@@ -4,7 +4,6 @@ import (
"encoding/json" "encoding/json"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os"
"testing" "testing"
"time" "time"
@@ -858,13 +857,16 @@ func TestWebhook_TriggerOnPause(t *testing.T) {
} }
func setupWebhookTest(t *testing.T, fn func(downloader *Downloader)) { func setupWebhookTest(t *testing.T, fn func(downloader *Downloader)) {
defaultDownloader.Setup() t.Helper()
defaultDownloader.cfg.StorageDir = ".test_storage" // Each test owns its runtime; a closed Downloader cannot be reused.
defaultDownloader.cfg.DownloadDir = ".test_download" d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
defer func() { if err := d.Setup(); err != nil {
defaultDownloader.Clear() t.Fatal(err)
os.RemoveAll(defaultDownloader.cfg.StorageDir) }
os.RemoveAll(defaultDownloader.cfg.DownloadDir) t.Cleanup(func() {
}() if err := d.Close(); err != nil {
fn(defaultDownloader) t.Error(err)
}
})
fn(d)
} }
-39
View File
@@ -1,9 +1,7 @@
package rest package rest
import ( import (
"io"
"net/http" "net/http"
"net/url"
"runtime" "runtime"
"github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/base"
@@ -337,43 +335,6 @@ func UpdateExtension(w http.ResponseWriter, r *http.Request) {
WriteJson(w, model.NewNilResult()) WriteJson(w, model.NewNilResult())
} }
func DoProxy(w http.ResponseWriter, r *http.Request) {
target := r.Header.Get("X-Target-Uri")
if target == "" {
writeError(w, "param invalid: X-Target-Uri")
return
}
targetUrl, err := url.Parse(target)
if err != nil {
writeError(w, err.Error())
return
}
r.RequestURI = ""
r.URL = targetUrl
r.Host = targetUrl.Host
r.Header.Del("Cookie")
r.Header.Del("X-Api-Token")
r.Header.Del("X-Target-Uri")
resp, err := http.DefaultClient.Do(r)
if err != nil {
writeError(w, err.Error())
return
}
defer resp.Body.Close()
for k, vv := range resp.Header {
for _, v := range vv {
w.Header().Set(k, v)
}
}
w.WriteHeader(resp.StatusCode)
buf, err := io.ReadAll(resp.Body)
if err != nil {
writeError(w, err.Error())
return
}
w.Write(buf)
}
func GetStats(w http.ResponseWriter, r *http.Request) { func GetStats(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
taskId := vars["id"] taskId := vars["id"]
+9
View File
@@ -0,0 +1,9 @@
package model
type MirrorCandidatesReq struct {
URL string `json:"url"`
}
type MirrorCandidatesResp struct {
URLs []string `json:"urls"`
}
+6 -3
View File
@@ -216,7 +216,6 @@ func buildServer(startCfg *model.StartConfig, continueOnStartup bool) (*http.Ser
WriteStatusJson(w, response.StatusCode, response.Body) WriteStatusJson(w, response.StatusCode, response.Body)
}) })
} }
r.Path("/api/web/proxy").HandlerFunc(DoProxy)
if startCfg.MCPEnable { if startCfg.MCPEnable {
r.Path("/mcp").Handler(mcpserver.NewHandler(Downloader)) r.Path("/mcp").Handler(mcpserver.NewHandler(Downloader))
} else { } else {
@@ -231,7 +230,6 @@ func buildServer(startCfg *model.StartConfig, continueOnStartup bool) (*http.Ser
if enableWebAuth { if enableWebAuth {
webSessions = newWebSessionStore() webSessions = newWebSessionStore()
} }
if startCfg.WebEnable {
if enableWebAuth { if enableWebAuth {
r.Methods(http.MethodPost).Path("/api/web/login").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { r.Methods(http.MethodPost).Path("/api/web/login").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var loginReq model.WebAuth var loginReq model.WebAuth
@@ -260,6 +258,11 @@ func buildServer(startCfg *model.StartConfig, continueOnStartup bool) (*http.Ser
WriteStatusJson(w, http.StatusUnauthorized, model.NewErrorResult("unauthorized", model.CodeUnauthorized)) WriteStatusJson(w, http.StatusUnauthorized, model.NewErrorResult("unauthorized", model.CodeUnauthorized))
}) })
} }
// Reserve the API namespace before the web file handler, including unknown
// endpoints and unsupported methods on otherwise known API paths.
r.Path("/api").Handler(http.NotFoundHandler())
r.PathPrefix("/api/").Handler(http.NotFoundHandler())
if startCfg.WebEnable {
r.PathPrefix("/fs/tasks").Handler(http.FileServer(new(taskFileSystem))) r.PathPrefix("/fs/tasks").Handler(http.FileServer(new(taskFileSystem)))
r.PathPrefix("/fs/extensions").Handler(http.FileServer(new(extensionFileSystem))) r.PathPrefix("/fs/extensions").Handler(http.FileServer(new(extensionFileSystem)))
r.PathPrefix("/").Handler(gzipMiddleware(http.FileServer(newEmbedCacheFileSystem(http.FS(startCfg.WebFS))))) r.PathPrefix("/").Handler(gzipMiddleware(http.FileServer(newEmbedCacheFileSystem(http.FS(startCfg.WebFS)))))
@@ -325,7 +328,7 @@ func buildServer(startCfg *model.StartConfig, continueOnStartup bool) (*http.Ser
}) })
server := &http.Server{Handler: handlers.CORS( server := &http.Server{Handler: handlers.CORS(
handlers.AllowedHeaders([]string{"Accept", "Content-Type", "Authorization", "X-Api-Token", "X-Target-Uri", "Mcp-Method", "Mcp-Name", "Mcp-Protocol-Version", "Mcp-Session-Id", "Last-Event-ID"}), handlers.AllowedHeaders([]string{"Accept", "Content-Type", "Authorization", "X-Api-Token", "Mcp-Method", "Mcp-Name", "Mcp-Protocol-Version", "Mcp-Session-Id", "Last-Event-ID"}),
handlers.AllowedMethods([]string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}), handlers.AllowedMethods([]string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}),
handlers.AllowedOrigins([]string{"*"}), handlers.AllowedOrigins([]string{"*"}),
handlers.ExposedHeaders([]string{"Mcp-Session-Id"}), handlers.ExposedHeaders([]string{"Mcp-Session-Id"}),
+40 -20
View File
@@ -2,7 +2,6 @@ package rest
import ( import (
"bytes" "bytes"
"crypto/md5"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@@ -15,6 +14,7 @@ import (
"strings" "strings"
"sync" "sync"
"testing" "testing"
"testing/fstest"
"time" "time"
"github.com/GopeedLab/gopeed/internal/test" "github.com/GopeedLab/gopeed/internal/test"
@@ -669,29 +669,49 @@ func TestWebFsEnhance(t *testing.T) {
}) })
} }
func TestDoProxy(t *testing.T) { func TestUnknownAPIPathsReturn404(t *testing.T) {
doTest(func() { for _, webEnabled := range []bool{false, true} {
code, respBody := doHttpRequest0(http.MethodGet, "/api/web/proxy", map[string]string{ t.Run(fmt.Sprintf("web=%t", webEnabled), func(t *testing.T) {
"X-Target-Uri": "https://github.com/GopeedLab/gopeed/raw/695da7ea87d2b455552b709d3cb4d7879484d4d1/README.md", doTest0(func(cfg *model.StartConfig) {
}, nil) cfg.WebEnable = webEnabled
if code != http.StatusOK { cfg.WebFS = fstest.MapFS{
t.Errorf("DoProxy() got = %v, want %v", code, http.StatusOK) "index.html": &fstest.MapFile{Data: []byte("<html>app</html>")},
"api/static.txt": &fstest.MapFile{Data: []byte("must not serve API paths as files")},
"apiary": &fstest.MapFile{Data: []byte("namespace boundary")},
} }
want := "4ee193b676f1ebb2ad810e016350d52a" }, func() {
got := fmt.Sprintf("%x", md5.Sum(respBody)) paths := []string{"/api", "/api/", "/api/missing", "/api/v1/missing", "/api/web/missing", "/api/web/login", "/api/static.txt"}
if got != want { for _, path := range paths {
t.Errorf("DoProxy() got = %v, want %v", got, want) for _, method := range []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete, http.MethodHead} {
} code, body := doHttpRequest0(method, path, nil, nil)
})
doTest(func() {
code, _ := doHttpRequest0(http.MethodGet, "/api/web/proxy", map[string]string{
"X-Target-Uri": "https://github.com/GopeedLab/gopeed/raw/695da7ea87d2b455552b709d3cb4d7879484d4d1/NOT_FOUND",
}, nil)
if code != http.StatusNotFound { if code != http.StatusNotFound {
t.Errorf("DoProxy() got = %v, want %v", code, http.StatusNotFound) t.Fatalf("%s %s status %d, want 404: %s", method, path, code, body)
}
if strings.Contains(string(body), "<html>app</html>") {
t.Fatalf("%s %s fell through to web files", method, path)
}
}
}
// Known API routes still run, while unsupported methods use the API fallback.
code, _ := doHttpRequest0(http.MethodGet, "/api/v1/info", nil, nil)
if code != http.StatusOK {
t.Fatalf("known API status %d, want 200", code)
}
code, _ = doHttpRequest0(http.MethodPost, "/api/v1/info", nil, nil)
if code != http.StatusNotFound {
t.Fatalf("unsupported API method status %d, want 404", code)
}
if webEnabled {
for path, want := range map[string]string{"/": "<html>app</html>", "/apiary": "namespace boundary"} {
code, body := doHttpRequest0(http.MethodGet, path, map[string]string{"Accept-Encoding": "identity"}, nil)
if code != http.StatusOK || !strings.Contains(string(body), want) {
t.Fatalf("web file %s status %d body %s", path, code, body)
}
}
} }
}) })
})
}
} }
func TestTestWebhook(t *testing.T) { func TestTestWebhook(t *testing.T) {
@@ -68,12 +68,11 @@ The following endpoints are web HTTP infrastructure and are not part of the FFI/
- `/fs/tasks/*` - `/fs/tasks/*`
- `/fs/extensions/*` - `/fs/extensions/*`
- `/api/web/proxy`
- Web static file hosting at `/` - Web static file hosting at `/`
`/fs/tasks` returns files from the Go host machine to a browser over HTTP. Native clients can access local file paths directly and do not need an equivalent API. `/fs/tasks` returns files from the Go host machine to a browser over HTTP. Native clients can access local file paths directly and do not need an equivalent API.
`/api/web/proxy` is infrastructure used by the web transport to work around browser CORS restrictions. It is not part of the published core `/api/v1/**` API. Native clients can request the target URL directly with Dio and do not need the Go proxy. All unmatched `/api` and `/api/**` requests use the common 404 handler before the web file handler.
## 4. Backend Maintenance Model ## 4. Backend Maintenance Model
@@ -182,7 +181,7 @@ The REST adapter is limited to transport responsibilities:
- Convert `http.Request` into a Dispatcher request. - Convert `http.Request` into a Dispatcher request.
- Write a Dispatcher response back to HTTP. - Write a Dispatcher response back to HTTP.
- Handle API tokens, web login, CORS, compression, and panic recovery. - Handle API tokens, web login, CORS, compression, and panic recovery.
- Register web-only file, proxy, and static resource routes. - Register web-only file and static resource routes.
The REST adapter no longer calls the global `rest.Downloader` directly. The REST adapter no longer calls the global `rest.Downloader` directly.
@@ -198,7 +197,6 @@ Core API routes
Web-only routes Web-only routes
├── /api/web/login ├── /api/web/login
├── /api/web/proxy
├── /fs/tasks/* ├── /fs/tasks/*
├── /fs/extensions/* ├── /fs/extensions/*
└── /* static Web UI └── /* static Web UI
@@ -416,12 +414,7 @@ The following patterns are prohibited:
### 7.4 Non-business HTTP capabilities ### 7.4 Non-business HTTP capabilities
`proxyRequest` is not a Gopeed Core API and belongs in a separate `ExternalHttpClient`: Tracker subscriptions run in Go. Application update checks and extension store requests run directly in the client. Go supplies mirror candidate URLs through `/api/v1/mirrors/{provider}/candidates`; clients probe and download update packages locally with Dio to retain real download progress. There is no arbitrary external HTTP forwarding API.
```text
Native → request the target directly with Dio
Web → /api/web/proxy
```
Task file access does not belong in `GopeedClient`: Task file access does not belong in `GopeedClient`:
@@ -594,7 +587,7 @@ Flutter converts all of them into `GopeedException`. Components do not handle Di
- Windows, macOS, Linux: startup, InvokeAsync, events, shutdown, and repeated startup. - Windows, macOS, Linux: startup, InvokeAsync, events, shutdown, and repeated startup.
- Android, iOS: gomobile InvokeAsync callbacks, reverse MethodChannel events, and background/foreground transitions. - Android, iOS: gomobile InvokeAsync callbacks, reverse MethodChannel events, and background/foreground transitions.
- Web: REST, login, proxy, and task-file downloads. - Web: REST, login, and task-file downloads.
- Native: enabling and disabling REST does not interrupt download tasks. - Native: enabling and disabling REST does not interrupt download tasks.
## 12. Implementation Phases ## 12. Implementation Phases
+13 -5
View File
@@ -208,11 +208,6 @@ Future<void> login(LoginReq loginReq) {
return _parse<void>(() => _request('api/web/login', method: 'POST', data: loginReq.toJson()), null); return _parse<void>(() => _request('api/web/login', method: 'POST', data: loginReq.toJson()), null);
} }
Future<Response<String>> proxyRequest<T>(String uri, {dynamic data, Options? options}) async {
initDefault();
return _transport!.proxyRequest(uri, data: data, options: options);
}
String join(String path) { String join(String path) {
initDefault(); initDefault();
return _transport!.join(path); return _transport!.join(path);
@@ -226,3 +221,16 @@ Future<dynamic> forward(
}) async { }) async {
return _request(path, method: method, data: data, queryParameters: queryParameters); return _request(path, method: method, data: data, queryParameters: queryParameters);
} }
Future<void> triggerProtocolJob(String protocol, String name) => _parse<void>(
() => _request(
'api/v1/protocols/${Uri.encodeComponent(protocol)}/jobs/${Uri.encodeComponent(name)}/trigger',
method: 'POST',
),
null,
);
Future<List<String>> getMirrorCandidates(String provider, String url) => _parse(
() => _request('api/v1/mirrors/${Uri.encodeComponent(provider)}/candidates', method: 'POST', data: {'url': url}),
(json) => List<String>.from(json['urls'] as List),
);
+28 -43
View File
@@ -1,27 +1,18 @@
import 'dart:convert';
import 'package:dio/dio.dart'; import 'package:dio/dio.dart';
import 'api.dart';
import 'model/store_extension.dart'; import 'model/store_extension.dart';
class GopeedSiteApi { class GopeedSiteApi {
GopeedSiteApi._(); GopeedSiteApi({Dio? client})
: _client =
client ??
Dio(BaseOptions(connectTimeout: const Duration(seconds: 10), receiveTimeout: const Duration(seconds: 10)));
static final instance = GopeedSiteApi._(); static final instance = GopeedSiteApi();
final Dio _client;
static const _host = 'gopeed.com';
final _storePages = <String, ({StoreExtensionPage page, DateTime expires})>{}; final _storePages = <String, ({StoreExtensionPage page, DateTime expires})>{};
final _storeRequests = <String, Future<StoreExtensionPage>>{}; final _storeRequests = <String, Future<StoreExtensionPage>>{};
Future<List<dynamic>> getReleases({int perPage = 10}) async {
final json = await _getJson('/api/releases', queryParameters: {'per_page': perPage.clamp(1, 100).toString()});
if (json is! List<dynamic>) {
throw const FormatException('Invalid releases response');
}
return json;
}
Future<StoreExtensionPage> getExtensions({ Future<StoreExtensionPage> getExtensions({
int page = 1, int page = 1,
int limit = 20, int limit = 20,
@@ -31,14 +22,14 @@ class GopeedSiteApi {
bool forceRefresh = false, bool forceRefresh = false,
}) async { }) async {
final params = { final params = {
'view': 'summary',
'page': page.toString(), 'page': page.toString(),
'limit': limit.clamp(1, 100).toString(), 'limit': limit.clamp(1, 100).toString(),
'sort': sort.name, 'sort': sort.name,
'order': order.name, 'order': order.name,
'view': 'summary',
if (query != null && query.trim().isNotEmpty) 'q': query.trim(), if (query != null && query.trim().isNotEmpty) 'q': query.trim(),
}; };
final key = Uri.https(_host, '/api/extensions', params).toString(); final key = Uri(queryParameters: params).toString();
final now = DateTime.now(); final now = DateTime.now();
_storePages.removeWhere((_, entry) => !now.isBefore(entry.expires)); _storePages.removeWhere((_, entry) => !now.isBefore(entry.expires));
final cached = _storePages[key]; final cached = _storePages[key];
@@ -58,40 +49,34 @@ class GopeedSiteApi {
} }
Future<StoreExtensionPage> _fetchStorePage(Map<String, String> params) async { Future<StoreExtensionPage> _fetchStorePage(Map<String, String> params) async {
final json = await _getJson('/api/extensions', queryParameters: params); final response = await _client.get<Map<String, dynamic>>(
return StoreExtensionPage.fromJson(json as Map<String, dynamic>); 'https://gopeed.com/api/extensions',
queryParameters: params,
);
return StoreExtensionPage.fromJson(response.data!);
} }
Future<StoreExtension> getExtension(String id, {String? version}) async { Future<StoreExtension> getExtension(String id, {String? version}) async {
final json = await _getJsonUri( final uri = Uri(
Uri(
scheme: 'https', scheme: 'https',
host: _host, host: 'gopeed.com',
pathSegments: ['api', 'extensions', id], pathSegments: ['api', 'extensions', id],
queryParameters: {if (version != null && version.isNotEmpty) 'version': version}, queryParameters: {'version': ?version},
),
); );
return StoreExtension.fromJson(json as Map<String, dynamic>); final response = await _client.get<Map<String, dynamic>>(uri.toString());
return StoreExtension.fromJson(response.data!);
}
Future<dynamic> getReleases() async {
final response = await _client.get<dynamic>(
'https://gopeed.com/api/releases',
queryParameters: {'per_page': 10},
options: Options(followRedirects: false),
);
return response.data;
} }
Future<void> reportExtensionInstall(String id) async { Future<void> reportExtensionInstall(String id) async {
final uri = Uri.https(_host, '/api/extensions/install'); await _client.post<void>('https://gopeed.com/api/extensions/install', data: {'id': id});
await proxyRequest(
uri.toString(),
data: {'id': id},
options: Options(method: 'POST', contentType: Headers.jsonContentType),
);
}
Future<dynamic> _getJson(String path, {Map<String, String>? queryParameters}) {
return _getJsonUri(Uri.https(_host, path, queryParameters));
}
Future<dynamic> _getJsonUri(Uri uri) async {
final Response<String> response = await proxyRequest(uri.toString());
if (response.data == null || response.data!.isEmpty) {
throw Exception('Empty response from $uri');
}
return jsonDecode(response.data!);
} }
} }
+43 -26
View File
@@ -8,6 +8,7 @@ class DownloaderConfig {
int maxRunning; int maxRunning;
ProtocolConfig protocolConfig = ProtocolConfig(); ProtocolConfig protocolConfig = ProtocolConfig();
ExtraConfig extra = ExtraConfig(); ExtraConfig extra = ExtraConfig();
GithubMirrorConfig githubMirror = GithubMirrorConfig();
ProxyConfig proxy = ProxyConfig(); ProxyConfig proxy = ProxyConfig();
WebhookConfig webhook = WebhookConfig(); WebhookConfig webhook = WebhookConfig();
ScriptConfig script = ScriptConfig(); ScriptConfig script = ScriptConfig();
@@ -98,10 +99,43 @@ class HttpConfig {
Map<String, dynamic> toJson() => _$HttpConfigToJson(this); Map<String, dynamic> toJson() => _$HttpConfigToJson(this);
} }
DateTime? _trackerUpdateTimeFromJson(String? value) =>
value == null || value.startsWith('0001-') ? null : DateTime.parse(value);
@JsonSerializable() @JsonSerializable()
class TrackerSubscriptionStatus {
final bool success;
final int trackerCount;
final DateTime lastFetchTime;
TrackerSubscriptionStatus({required this.success, required this.trackerCount, required this.lastFetchTime});
factory TrackerSubscriptionStatus.fromJson(Map<String, dynamic> json) => _$TrackerSubscriptionStatusFromJson(json);
Map<String, dynamic> toJson() => _$TrackerSubscriptionStatusToJson(this);
}
@JsonSerializable(explicitToJson: true)
class BtState {
Map<String, List<String>> trackerSources;
Map<String, TrackerSubscriptionStatus> trackerSubscriptionStatuses;
@JsonKey(fromJson: _trackerUpdateTimeFromJson)
DateTime? lastTrackerUpdateTime;
BtState({this.trackerSources = const {}, this.trackerSubscriptionStatuses = const {}, this.lastTrackerUpdateTime});
factory BtState.fromJson(Map<String, dynamic>? json) => json == null ? BtState() : _$BtStateFromJson(json);
Map<String, dynamic> toJson() => _$BtStateToJson(this);
}
@JsonSerializable(explicitToJson: true)
class BtConfig { class BtConfig {
BtState state = BtState();
List<String> trackerSubscribeUrls;
bool autoUpdateTrackers;
List<String> customTrackers;
int listenPort; int listenPort;
List<String> trackers;
bool seedKeep; bool seedKeep;
double seedRatio; double seedRatio;
int seedTime; int seedTime;
@@ -109,8 +143,10 @@ class BtConfig {
bool disablePex; bool disablePex;
BtConfig({ BtConfig({
this.trackerSubscribeUrls = const [],
this.autoUpdateTrackers = true,
this.customTrackers = const [],
this.listenPort = 0, this.listenPort = 0,
this.trackers = const [],
this.seedKeep = false, this.seedKeep = false,
this.seedRatio = 0, this.seedRatio = 0,
this.seedTime = 0, this.seedTime = 0,
@@ -158,9 +194,6 @@ class ExtraConfig {
String analyticsClientId; String analyticsClientId;
List<DownloadCategory> downloadCategories; List<DownloadCategory> downloadCategories;
ExtraConfigBt bt = ExtraConfigBt();
ExtraConfigGithubMirror githubMirror = ExtraConfigGithubMirror();
ExtraConfig({ ExtraConfig({
this.themeMode = '', this.themeMode = '',
this.themeColor = 'green', this.themeColor = 'green',
@@ -271,22 +304,6 @@ class ProxyConfig {
Map<String, dynamic> toJson() => _$ProxyConfigToJson(this); Map<String, dynamic> toJson() => _$ProxyConfigToJson(this);
} }
@JsonSerializable()
class ExtraConfigBt {
List<String> trackerSubscribeUrls = [];
List<String> subscribeTrackers = [];
bool autoUpdateTrackers = true;
DateTime? lastTrackerUpdateTime;
List<String> customTrackers = [];
ExtraConfigBt();
factory ExtraConfigBt.fromJson(Map<String, dynamic> json) => _$ExtraConfigBtFromJson(json);
Map<String, dynamic> toJson() => _$ExtraConfigBtToJson(this);
}
enum GithubMirrorType { jsdelivr, ghProxy } enum GithubMirrorType { jsdelivr, ghProxy }
@JsonSerializable() @JsonSerializable()
@@ -304,16 +321,16 @@ class GithubMirror {
} }
@JsonSerializable(explicitToJson: true) @JsonSerializable(explicitToJson: true)
class ExtraConfigGithubMirror { class GithubMirrorConfig {
bool enabled; bool enabled;
List<GithubMirror> mirrors; List<GithubMirror> mirrors;
ExtraConfigGithubMirror({this.enabled = true, this.mirrors = const []}); GithubMirrorConfig({this.enabled = true, this.mirrors = const []});
factory ExtraConfigGithubMirror.fromJson(Map<String, dynamic>? json) => factory GithubMirrorConfig.fromJson(Map<String, dynamic>? json) =>
json == null ? ExtraConfigGithubMirror() : _$ExtraConfigGithubMirrorFromJson(json); json == null ? GithubMirrorConfig() : _$GithubMirrorConfigFromJson(json);
Map<String, dynamic> toJson() => _$ExtraConfigGithubMirrorToJson(this); Map<String, dynamic> toJson() => _$GithubMirrorConfigToJson(this);
} }
@JsonSerializable() @JsonSerializable()
@@ -15,6 +15,7 @@ DownloaderConfig _$DownloaderConfigFromJson(Map<String, dynamic> json) =>
) )
..protocolConfig = ProtocolConfig.fromJson(json['protocolConfig'] as Map<String, dynamic>?) ..protocolConfig = ProtocolConfig.fromJson(json['protocolConfig'] as Map<String, dynamic>?)
..extra = ExtraConfig.fromJson(json['extra'] as Map<String, dynamic>?) ..extra = ExtraConfig.fromJson(json['extra'] as Map<String, dynamic>?)
..githubMirror = GithubMirrorConfig.fromJson(json['githubMirror'] as Map<String, dynamic>?)
..proxy = ProxyConfig.fromJson(json['proxy'] as Map<String, dynamic>) ..proxy = ProxyConfig.fromJson(json['proxy'] as Map<String, dynamic>)
..webhook = WebhookConfig.fromJson(json['webhook'] as Map<String, dynamic>?) ..webhook = WebhookConfig.fromJson(json['webhook'] as Map<String, dynamic>?)
..script = ScriptConfig.fromJson(json['script'] as Map<String, dynamic>?) ..script = ScriptConfig.fromJson(json['script'] as Map<String, dynamic>?)
@@ -27,6 +28,7 @@ Map<String, dynamic> _$DownloaderConfigToJson(DownloaderConfig instance) => <Str
'maxRunning': instance.maxRunning, 'maxRunning': instance.maxRunning,
'protocolConfig': instance.protocolConfig.toJson(), 'protocolConfig': instance.protocolConfig.toJson(),
'extra': instance.extra.toJson(), 'extra': instance.extra.toJson(),
'githubMirror': instance.githubMirror.toJson(),
'proxy': instance.proxy.toJson(), 'proxy': instance.proxy.toJson(),
'webhook': instance.webhook.toJson(), 'webhook': instance.webhook.toJson(),
'script': instance.script.toJson(), 'script': instance.script.toJson(),
@@ -92,19 +94,56 @@ Map<String, dynamic> _$HttpConfigToJson(HttpConfig instance) => <String, dynamic
'useServerCtime': instance.useServerCtime, 'useServerCtime': instance.useServerCtime,
}; };
TrackerSubscriptionStatus _$TrackerSubscriptionStatusFromJson(Map<String, dynamic> json) => TrackerSubscriptionStatus(
success: json['success'] as bool,
trackerCount: (json['trackerCount'] as num).toInt(),
lastFetchTime: DateTime.parse(json['lastFetchTime'] as String),
);
Map<String, dynamic> _$TrackerSubscriptionStatusToJson(TrackerSubscriptionStatus instance) => <String, dynamic>{
'success': instance.success,
'trackerCount': instance.trackerCount,
'lastFetchTime': instance.lastFetchTime.toIso8601String(),
};
BtState _$BtStateFromJson(Map<String, dynamic> json) => BtState(
trackerSources:
(json['trackerSources'] as Map<String, dynamic>?)?.map(
(k, e) => MapEntry(k, (e as List<dynamic>).map((e) => e as String).toList()),
) ??
const {},
trackerSubscriptionStatuses:
(json['trackerSubscriptionStatuses'] as Map<String, dynamic>?)?.map(
(k, e) => MapEntry(k, TrackerSubscriptionStatus.fromJson(e as Map<String, dynamic>)),
) ??
const {},
lastTrackerUpdateTime: _trackerUpdateTimeFromJson(json['lastTrackerUpdateTime'] as String?),
);
Map<String, dynamic> _$BtStateToJson(BtState instance) => <String, dynamic>{
'trackerSources': instance.trackerSources,
'trackerSubscriptionStatuses': instance.trackerSubscriptionStatuses.map((k, e) => MapEntry(k, e.toJson())),
'lastTrackerUpdateTime': ?instance.lastTrackerUpdateTime?.toIso8601String(),
};
BtConfig _$BtConfigFromJson(Map<String, dynamic> json) => BtConfig( BtConfig _$BtConfigFromJson(Map<String, dynamic> json) => BtConfig(
trackerSubscribeUrls: (json['trackerSubscribeUrls'] as List<dynamic>?)?.map((e) => e as String).toList() ?? const [],
autoUpdateTrackers: json['autoUpdateTrackers'] as bool? ?? true,
customTrackers: (json['customTrackers'] as List<dynamic>?)?.map((e) => e as String).toList() ?? const [],
listenPort: (json['listenPort'] as num?)?.toInt() ?? 0, listenPort: (json['listenPort'] as num?)?.toInt() ?? 0,
trackers: (json['trackers'] as List<dynamic>?)?.map((e) => e as String).toList() ?? const [],
seedKeep: json['seedKeep'] as bool? ?? false, seedKeep: json['seedKeep'] as bool? ?? false,
seedRatio: (json['seedRatio'] as num?)?.toDouble() ?? 0, seedRatio: (json['seedRatio'] as num?)?.toDouble() ?? 0,
seedTime: (json['seedTime'] as num?)?.toInt() ?? 0, seedTime: (json['seedTime'] as num?)?.toInt() ?? 0,
disableDht: json['disableDht'] as bool? ?? false, disableDht: json['disableDht'] as bool? ?? false,
disablePex: json['disablePex'] as bool? ?? false, disablePex: json['disablePex'] as bool? ?? false,
); )..state = BtState.fromJson(json['state'] as Map<String, dynamic>?);
Map<String, dynamic> _$BtConfigToJson(BtConfig instance) => <String, dynamic>{ Map<String, dynamic> _$BtConfigToJson(BtConfig instance) => <String, dynamic>{
'state': instance.state.toJson(),
'trackerSubscribeUrls': instance.trackerSubscribeUrls,
'autoUpdateTrackers': instance.autoUpdateTrackers,
'customTrackers': instance.customTrackers,
'listenPort': instance.listenPort, 'listenPort': instance.listenPort,
'trackers': instance.trackers,
'seedKeep': instance.seedKeep, 'seedKeep': instance.seedKeep,
'seedRatio': instance.seedRatio, 'seedRatio': instance.seedRatio,
'seedTime': instance.seedTime, 'seedTime': instance.seedTime,
@@ -128,8 +167,7 @@ Map<String, dynamic> _$Ed2kConfigToJson(Ed2kConfig instance) => <String, dynamic
'nodesDat': instance.nodesDat, 'nodesDat': instance.nodesDat,
}; };
ExtraConfig _$ExtraConfigFromJson(Map<String, dynamic> json) => ExtraConfig _$ExtraConfigFromJson(Map<String, dynamic> json) => ExtraConfig(
ExtraConfig(
themeMode: json['themeMode'] as String? ?? '', themeMode: json['themeMode'] as String? ?? '',
themeColor: json['themeColor'] as String? ?? 'green', themeColor: json['themeColor'] as String? ?? 'green',
locale: json['locale'] as String? ?? '', locale: json['locale'] as String? ?? '',
@@ -139,10 +177,7 @@ ExtraConfig _$ExtraConfigFromJson(Map<String, dynamic> json) =>
notifyWhenNewVersion: json['notifyWhenNewVersion'] as bool? ?? true, notifyWhenNewVersion: json['notifyWhenNewVersion'] as bool? ?? true,
desktopNotification: json['desktopNotification'] as bool? ?? true, desktopNotification: json['desktopNotification'] as bool? ?? true,
backgroundLocationKeepAlive: json['backgroundLocationKeepAlive'] as bool? ?? false, backgroundLocationKeepAlive: json['backgroundLocationKeepAlive'] as bool? ?? false,
backgroundContinuedProcessing: backgroundContinuedProcessing: json['backgroundContinuedProcessing'] as bool? ?? false,
json['backgroundContinuedProcessing']
as bool? ??
false,
windowState: json['windowState'] == null windowState: json['windowState'] == null
? null ? null
: WindowStateConfig.fromJson(json['windowState'] as Map<String, dynamic>?), : WindowStateConfig.fromJson(json['windowState'] as Map<String, dynamic>?),
@@ -156,9 +191,7 @@ ExtraConfig _$ExtraConfigFromJson(Map<String, dynamic> json) =>
?.map((e) => DownloadCategory.fromJson(e as Map<String, dynamic>)) ?.map((e) => DownloadCategory.fromJson(e as Map<String, dynamic>))
.toList() ?? .toList() ??
const [], const [],
) );
..bt = ExtraConfigBt.fromJson(json['bt'] as Map<String, dynamic>)
..githubMirror = ExtraConfigGithubMirror.fromJson(json['githubMirror'] as Map<String, dynamic>?);
Map<String, dynamic> _$ExtraConfigToJson(ExtraConfig instance) => <String, dynamic>{ Map<String, dynamic> _$ExtraConfigToJson(ExtraConfig instance) => <String, dynamic>{
'themeMode': instance.themeMode, 'themeMode': instance.themeMode,
@@ -170,8 +203,7 @@ Map<String, dynamic> _$ExtraConfigToJson(ExtraConfig instance) => <String, dynam
'notifyWhenNewVersion': instance.notifyWhenNewVersion, 'notifyWhenNewVersion': instance.notifyWhenNewVersion,
'desktopNotification': instance.desktopNotification, 'desktopNotification': instance.desktopNotification,
'backgroundLocationKeepAlive': instance.backgroundLocationKeepAlive, 'backgroundLocationKeepAlive': instance.backgroundLocationKeepAlive,
'backgroundContinuedProcessing': 'backgroundContinuedProcessing': instance.backgroundContinuedProcessing,
instance.backgroundContinuedProcessing,
'windowState': instance.windowState.toJson(), 'windowState': instance.windowState.toJson(),
'bookmarks': instance.bookmarks, 'bookmarks': instance.bookmarks,
'createHistory': instance.createHistory, 'createHistory': instance.createHistory,
@@ -179,8 +211,6 @@ Map<String, dynamic> _$ExtraConfigToJson(ExtraConfig instance) => <String, dynam
'analyticsEnabled': instance.analyticsEnabled, 'analyticsEnabled': instance.analyticsEnabled,
'analyticsClientId': instance.analyticsClientId, 'analyticsClientId': instance.analyticsClientId,
'downloadCategories': instance.downloadCategories.map((e) => e.toJson()).toList(), 'downloadCategories': instance.downloadCategories.map((e) => e.toJson()).toList(),
'bt': instance.bt.toJson(),
'githubMirror': instance.githubMirror.toJson(),
}; };
WindowStateConfig _$WindowStateConfigFromJson(Map<String, dynamic> json) => WindowStateConfig( WindowStateConfig _$WindowStateConfigFromJson(Map<String, dynamic> json) => WindowStateConfig(
@@ -249,23 +279,6 @@ Map<String, dynamic> _$ProxyConfigToJson(ProxyConfig instance) => <String, dynam
'pwd': instance.pwd, 'pwd': instance.pwd,
}; };
ExtraConfigBt _$ExtraConfigBtFromJson(Map<String, dynamic> json) => ExtraConfigBt()
..trackerSubscribeUrls = (json['trackerSubscribeUrls'] as List<dynamic>).map((e) => e as String).toList()
..subscribeTrackers = (json['subscribeTrackers'] as List<dynamic>).map((e) => e as String).toList()
..autoUpdateTrackers = json['autoUpdateTrackers'] as bool
..lastTrackerUpdateTime = json['lastTrackerUpdateTime'] == null
? null
: DateTime.parse(json['lastTrackerUpdateTime'] as String)
..customTrackers = (json['customTrackers'] as List<dynamic>).map((e) => e as String).toList();
Map<String, dynamic> _$ExtraConfigBtToJson(ExtraConfigBt instance) => <String, dynamic>{
'trackerSubscribeUrls': instance.trackerSubscribeUrls,
'subscribeTrackers': instance.subscribeTrackers,
'autoUpdateTrackers': instance.autoUpdateTrackers,
'lastTrackerUpdateTime': ?instance.lastTrackerUpdateTime?.toIso8601String(),
'customTrackers': instance.customTrackers,
};
GithubMirror _$GithubMirrorFromJson(Map<String, dynamic> json) => GithubMirror( GithubMirror _$GithubMirrorFromJson(Map<String, dynamic> json) => GithubMirror(
type: $enumDecode(_$GithubMirrorTypeEnumMap, json['type']), type: $enumDecode(_$GithubMirrorTypeEnumMap, json['type']),
url: json['url'] as String, url: json['url'] as String,
@@ -282,14 +295,14 @@ Map<String, dynamic> _$GithubMirrorToJson(GithubMirror instance) => <String, dyn
const _$GithubMirrorTypeEnumMap = {GithubMirrorType.jsdelivr: 'jsdelivr', GithubMirrorType.ghProxy: 'ghProxy'}; const _$GithubMirrorTypeEnumMap = {GithubMirrorType.jsdelivr: 'jsdelivr', GithubMirrorType.ghProxy: 'ghProxy'};
ExtraConfigGithubMirror _$ExtraConfigGithubMirrorFromJson(Map<String, dynamic> json) => ExtraConfigGithubMirror( GithubMirrorConfig _$GithubMirrorConfigFromJson(Map<String, dynamic> json) => GithubMirrorConfig(
enabled: json['enabled'] as bool? ?? true, enabled: json['enabled'] as bool? ?? true,
mirrors: mirrors:
(json['mirrors'] as List<dynamic>?)?.map((e) => GithubMirror.fromJson(e as Map<String, dynamic>)).toList() ?? (json['mirrors'] as List<dynamic>?)?.map((e) => GithubMirror.fromJson(e as Map<String, dynamic>)).toList() ??
const [], const [],
); );
Map<String, dynamic> _$ExtraConfigGithubMirrorToJson(ExtraConfigGithubMirror instance) => <String, dynamic>{ Map<String, dynamic> _$GithubMirrorConfigToJson(GithubMirrorConfig instance) => <String, dynamic>{
'enabled': instance.enabled, 'enabled': instance.enabled,
'mirrors': instance.mirrors.map((e) => e.toJson()).toList(), 'mirrors': instance.mirrors.map((e) => e.toJson()).toList(),
}; };
@@ -317,12 +317,7 @@ class AppPlatformController extends AsyncNotifier<AppPlatformState> with WindowL
} }
Future<void> startUpdate(VersionInfo versionInfo, UpdateProgressCallback onProgress) async { Future<void> startUpdate(VersionInfo versionInfo, UpdateProgressCallback onProgress) async {
final runtime = ref.read(appRuntimeControllerProvider).value; await updateApp(versionInfo, service: ref.read(gopeedServiceProvider), onProgress: onProgress);
await updateApp(
versionInfo,
githubMirror: runtime?.downloaderConfig.extra.githubMirror ?? ExtraConfigGithubMirror(enabled: false),
onProgress: onProgress,
);
} }
AppPlatformState _snapshot() => AppPlatformState( AppPlatformState _snapshot() => AppPlatformState(
@@ -1,5 +1,4 @@
import 'dart:async'; import 'dart:async';
import 'dart:convert';
import 'package:flutter/foundation.dart'; import 'package:flutter/foundation.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
@@ -106,14 +105,9 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
), ),
); );
unawaited(LocationKeepAliveCoordinator.instance.reconcile(enabled: config.extra.backgroundLocationKeepAlive)); unawaited(LocationKeepAliveCoordinator.instance.reconcile(enabled: config.extra.backgroundLocationKeepAlive));
unawaited( unawaited(ContinuedProcessing.setEnabled(config.extra.backgroundContinuedProcessing));
ContinuedProcessing.setEnabled(
config.extra.backgroundContinuedProcessing,
),
);
} }
Future<AppRuntimeState> _init() async { Future<AppRuntimeState> _init() async {
await AppInitializer.ensureStorageInitialized(); await AppInitializer.ensureStorageInitialized();
try { try {
@@ -175,7 +169,6 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
..address = config.api.address ..address = config.api.address
..apiToken = config.api.token; ..apiToken = config.api.token;
} }
unawaited(_initTrackerUpdate(config));
unawaited(_initAndroidForegroundService(appLocalizationsFor(config.extra.locale))); unawaited(_initAndroidForegroundService(appLocalizationsFor(config.extra.locale)));
await _initAnalytics(config); await _initAnalytics(config);
@@ -190,9 +183,7 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
// Make sure the native Continued Processing manager has the // Make sure the native Continued Processing manager has the
// persisted setting before app initialization completes. This // persisted setting before app initialization completes. This
// avoids the first download racing ahead of setEnabled(true). // avoids the first download racing ahead of setEnabled(true).
await ContinuedProcessing.setEnabled( await ContinuedProcessing.setEnabled(config.extra.backgroundContinuedProcessing);
config.extra.backgroundContinuedProcessing,
);
LocationKeepAliveCoordinator.instance.start( LocationKeepAliveCoordinator.instance.start(
() => () =>
@@ -271,9 +262,6 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
if (extra.themeColor.isEmpty) { if (extra.themeColor.isEmpty) {
extra.themeColor = 'green'; extra.themeColor = 'green';
} }
if (extra.bt.trackerSubscribeUrls.isEmpty) {
extra.bt.trackerSubscribeUrls.addAll(allTrackerSubscribeUrls);
}
if (config.proxy.scheme.isEmpty) { if (config.proxy.scheme.isEmpty) {
config.proxy.scheme = 'http'; config.proxy.scheme = 'http';
} }
@@ -308,26 +296,6 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
), ),
]; ];
} }
if (extra.githubMirror.mirrors.isEmpty) {
extra.githubMirror.mirrors = [
GithubMirror(type: GithubMirrorType.jsdelivr, url: 'https://fastly.jsdelivr.net/gh', isBuiltIn: true),
GithubMirror(type: GithubMirrorType.ghProxy, url: 'https://fastgit.cc', isBuiltIn: true),
];
}
}
Future<void> _initTrackerUpdate(DownloaderConfig config) async {
final btExtra = config.extra.bt;
final lastUpdateTime = btExtra.lastTrackerUpdateTime;
if (!btExtra.autoUpdateTrackers ||
(lastUpdateTime != null && !lastUpdateTime.isBefore(DateTime.now().subtract(const Duration(days: 1))))) {
return;
}
try {
await updateTrackers(config);
} catch (error, stackTrace) {
logger.w('tracker update failed', error, stackTrace);
}
} }
Future<void> _initAnalytics(DownloaderConfig config) async { Future<void> _initAnalytics(DownloaderConfig config) async {
@@ -344,32 +312,12 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
} }
} }
Future<void> updateTrackers(DownloaderConfig config) async { Future<void> updateTrackers() async {
final btExtra = config.extra.bt; final service = ref.read(gopeedServiceProvider);
final trackers = <String>[]; try {
for (final url in btExtra.trackerSubscribeUrls) { await service.triggerProtocolJob('bt', 'update-trackers');
final resp = await api.proxyRequest(url); } finally {
final data = resp.data; replaceDownloaderConfig(await service.getConfig());
if (resp.statusCode == 200 && data != null && data.isNotEmpty) {
trackers.addAll(const LineSplitter().convert(data).where((line) => line.trim().isNotEmpty));
}
}
btExtra.subscribeTrackers = trackers;
btExtra.lastTrackerUpdateTime = DateTime.now();
config.protocolConfig.bt.trackers = {...btExtra.subscribeTrackers, ...btExtra.customTrackers}.toList();
await ref.read(gopeedServiceProvider).putConfig(config);
final current = state.value;
if (current != null) {
state = AsyncValue.data(
AppRuntimeState(
startConfig: current.startConfig,
apiServerState: current.apiServerState,
downloaderConfig: config,
localBackendStarted: current.localBackendStarted,
startupError: current.startupError,
),
);
} }
} }
@@ -509,17 +457,3 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
return operation; return operation;
} }
} }
const allTrackerSubscribeUrls = [
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt',
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt',
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_https.txt',
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt',
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_udp.txt',
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ws.txt',
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best.txt',
'https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best_ip.txt',
'https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/all.txt',
'https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best.txt',
'https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/http.txt',
];
@@ -82,6 +82,14 @@ RpcCodecRegistry createAppCapabilityCodecs() {
void _bindGopeed(CapabilityRegistry registry) { void _bindGopeed(CapabilityRegistry registry) {
registry registry
..bind(GopeedMethods.triggerProtocolJob, (p) async {
await api.triggerProtocolJob(p['protocol'] as String, p['name'] as String);
return const RpcUnit();
})
..bind(
GopeedMethods.getMirrorCandidates,
(p) => api.getMirrorCandidates(p['provider'] as String, p['url'] as String),
)
..bind(GopeedMethods.resolve, api.resolve) ..bind(GopeedMethods.resolve, api.resolve)
..bind(GopeedMethods.createTask, api.createTask) ..bind(GopeedMethods.createTask, api.createTask)
..bind(GopeedMethods.createTaskBatch, api.createTaskBatch) ..bind(GopeedMethods.createTaskBatch, api.createTaskBatch)
@@ -13,6 +13,8 @@ import '../../api/model/update_extension_settings.dart';
import 'capability_rpc.dart'; import 'capability_rpc.dart';
abstract final class GopeedMethods { abstract final class GopeedMethods {
static const triggerProtocolJob = RpcMethod<Map<String, dynamic>, RpcUnit>('gopeed.protocol.job.trigger');
static const getMirrorCandidates = RpcMethod<Map<String, dynamic>, List<String>>('gopeed.mirrors.candidates');
static const resolve = RpcMethod<ResolveTask, ResolveResult>('gopeed.resolve'); static const resolve = RpcMethod<ResolveTask, ResolveResult>('gopeed.resolve');
static const createTask = RpcMethod<CreateTask, String>('gopeed.task.create'); static const createTask = RpcMethod<CreateTask, String>('gopeed.task.create');
static const createTaskBatch = RpcMethod<CreateTaskBatch, List<String>>('gopeed.task.createBatch'); static const createTaskBatch = RpcMethod<CreateTaskBatch, List<String>>('gopeed.task.createBatch');
@@ -43,6 +45,10 @@ class GopeedService {
final CapabilityInvoker _invoker; final CapabilityInvoker _invoker;
Future<void> triggerProtocolJob(String protocol, String name) async =>
_invoker.invoke(GopeedMethods.triggerProtocolJob, {'protocol': protocol, 'name': name});
Future<List<String>> getMirrorCandidates(String provider, String url) =>
_invoker.invoke(GopeedMethods.getMirrorCandidates, {'provider': provider, 'url': url});
Future<ResolveResult> resolve(ResolveTask request) => _invoker.invoke(GopeedMethods.resolve, request); Future<ResolveResult> resolve(ResolveTask request) => _invoker.invoke(GopeedMethods.resolve, request);
Future<String> createTask(CreateTask request) => _invoker.invoke(GopeedMethods.createTask, request); Future<String> createTask(CreateTask request) => _invoker.invoke(GopeedMethods.createTask, request);
@@ -0,0 +1,147 @@
import 'package:dio/dio.dart';
import 'package:pub_semver/pub_semver.dart';
import '../../api/gopeed_site_api.dart';
class VersionInfo {
const VersionInfo({
required this.version,
required this.changeLog,
required this.releaseUrl,
this.assetSha256 = const {},
});
final String version;
final String changeLog;
final String releaseUrl;
// Populated only from official release metadata, never from a download mirror.
final Map<String, String> assetSha256;
}
/// Query the official GitHub API directly, then the independent website API
/// only if the GitHub request or its response validation fails.
class AppUpdateClient {
AppUpdateClient({Dio? client, GopeedSiteApi? site})
: _client =
client ??
Dio(BaseOptions(connectTimeout: const Duration(seconds: 10), receiveTimeout: const Duration(seconds: 30))),
_site = site ?? GopeedSiteApi.instance;
static const releasesUrl = 'https://api.github.com/repos/GopeedLab/gopeed/releases?per_page=10';
final Dio _client;
final GopeedSiteApi _site;
Future<List<_Release>> _fetchGithubReleases() async {
final cancel = CancelToken();
try {
final response = await _client
.get<dynamic>(releasesUrl, cancelToken: cancel, options: Options(followRedirects: false))
.timeout(const Duration(seconds: 30));
return _decodeReleases(response.data);
} finally {
cancel.cancel('GitHub release request completed');
}
}
Future<VersionInfo?> check(String currentVersion) async {
final current = _parseVersion(currentVersion);
if (current == null) throw FormatException('Invalid current application version', currentVersion);
List<_Release> releases;
try {
releases = await _fetchGithubReleases();
} catch (_) {
releases = _decodeReleases(await _site.getReleases());
}
_Release? selected;
for (final release in releases) {
final version = release.version;
if (release.draft || version.compareTo(current) <= 0) continue;
final preview = release.prerelease || version.isPreRelease;
if (!current.isPreRelease && preview) continue;
if (current.isPreRelease &&
preview &&
(version.major != current.major || version.minor != current.minor || version.patch != current.patch)) {
continue;
}
if (selected == null || version.compareTo(selected.version) > 0) selected = release;
}
if (selected == null) return null;
return VersionInfo(
version: selected.tag.replaceFirst(RegExp(r'^v'), ''),
changeLog: selected.body,
assetSha256: Map.unmodifiable(selected.assetSha256),
releaseUrl: selected.url.isEmpty
? 'https://github.com/GopeedLab/gopeed/releases/tag/${selected.tag}'
: selected.url,
);
}
}
// Build metadata does not affect SemVer precedence.
Version? _parseVersion(String raw) {
try {
final version = Version.parse(raw.replaceFirst(RegExp(r'^v'), ''));
return Version(version.major, version.minor, version.patch, pre: version.preRelease.join('.'));
} on FormatException {
return null;
}
}
List<_Release> _decodeReleases(dynamic data) {
if (data is! List) throw const FormatException('Invalid release list');
final releases = <_Release>[];
for (final entry in data) {
if (entry is! Map || entry['tag_name'] is! String) continue;
final tag = entry['tag_name'] as String;
final version = _parseVersion(tag);
if (version == null) continue;
releases.add(
_Release(
tag: tag,
version: version,
draft: entry['draft'] == true,
prerelease: entry['prerelease'] == true,
body: entry['body'] as String? ?? '',
url: entry['html_url'] as String? ?? '',
assetSha256: _decodeAssetSha256(entry['assets']),
),
);
}
if (data.isNotEmpty && releases.isEmpty) throw const FormatException('Response contains no valid releases');
return releases;
}
Map<String, String> _decodeAssetSha256(dynamic assets) {
if (assets == null) return {};
if (assets is! List) throw const FormatException('Invalid release assets');
final result = <String, String>{};
for (final asset in assets) {
if (asset is! Map || asset['name'] is! String || asset['digest'] is! String) continue;
final digest = asset['digest'] as String;
final match = RegExp(r'^sha256:([a-fA-F0-9]{64})$').firstMatch(digest);
if (match == null || match.end != digest.length) continue;
final name = asset['name'] as String;
if (result.containsKey(name)) throw const FormatException('Duplicate release asset');
result[name] = match.group(1)!.toLowerCase();
}
return result;
}
class _Release {
const _Release({
required this.tag,
required this.version,
required this.draft,
required this.prerelease,
required this.body,
required this.url,
required this.assetSha256,
});
final String tag;
final Version version;
final bool draft;
final bool prerelease;
final String body;
final String url;
final Map<String, String> assetSha256;
}
@@ -0,0 +1,92 @@
import 'dart:io';
import 'package:crypto/crypto.dart';
import 'package:dio/dio.dart';
class DownloadChecksumException implements Exception {
const DownloadChecksumException();
@override
String toString() => 'Update package checksum verification failed';
}
/// Probes backend-provided URLs concurrently, then downloads the fastest usable
/// candidate. Other candidates remain available if HEAD is unsupported or the
/// actual download fails. This layer has no GitHub URL or mirror-config rules.
Future<void> downloadFromCandidates(
List<String> candidates,
String savePath, {
required ProgressCallback onReceiveProgress,
required String expectedSha256,
Dio? client,
Duration probeTimeout = const Duration(seconds: 3),
}) async {
// Reject missing or malformed trusted metadata before any network request.
if (expectedSha256.length != 64 || !RegExp(r'^[a-fA-F0-9]{64}$').hasMatch(expectedSha256)) {
throw const DownloadChecksumException();
}
final urls = candidates.toSet().toList(growable: false);
if (urls.isEmpty) throw ArgumentError.value(candidates, 'candidates', 'No download candidates');
final dio = client ?? Dio(BaseOptions(connectTimeout: const Duration(seconds: 10)));
try {
final probes = await Future.wait(
urls.map((url) async {
final watch = Stopwatch()..start();
final cancel = CancelToken();
try {
final response = await dio
.head<void>(
url,
cancelToken: cancel,
options: Options(receiveTimeout: probeTimeout, validateStatus: (status) => status == 200),
)
.timeout(probeTimeout);
if (_isPackageResponse(response.headers)) return (url: url, latency: watch.elapsed);
} catch (_) {
// HEAD failure does not prove that a normal GET will fail.
} finally {
cancel.cancel();
}
return (url: url, latency: null);
}),
);
final available = probes.where((probe) => probe.latency != null).toList()
..sort((a, b) => a.latency!.compareTo(b.latency!));
final ordered = [
...available.map((probe) => probe.url),
...probes.where((p) => p.latency == null).map((p) => p.url),
];
Object? lastError;
StackTrace? lastStack;
for (final url in ordered) {
try {
onReceiveProgress(0, 0);
final response = await dio.download(
url,
savePath,
onReceiveProgress: onReceiveProgress,
deleteOnError: true,
options: Options(receiveTimeout: const Duration(seconds: 30), validateStatus: (status) => status == 200),
);
if (!_isPackageResponse(response.headers) || await File(savePath).length() == 0) {
throw StateError('Invalid download response from $url');
}
final actual = await sha256.bind(File(savePath).openRead()).first;
if (actual.toString() != expectedSha256.toLowerCase()) throw const DownloadChecksumException();
return;
} catch (error, stack) {
lastError = error;
lastStack = stack;
final file = File(savePath);
if (await file.exists()) await file.delete();
}
}
Error.throwWithStackTrace(lastError!, lastStack!);
} finally {
if (client == null) dio.close(force: true);
}
}
bool _isPackageResponse(Headers headers) {
final contentType = headers.value(Headers.contentTypeHeader)?.split(';').first.trim().toLowerCase();
return contentType != 'text/html' && contentType != 'application/xhtml+xml' && contentType != 'application/json';
}
Loaded 50 of 98 files, more files were not shown because too many files have changed in this diff. Show more