refactor: remove proxy API (#1596)

This commit is contained in:
Levi authored and GitHub committed 2026-10-21 22:48:17 +08:00
1 parent 47cafb7575
commit 4214765d39
98 files changed
+5958 -1225

No files matched your search

+72
View File
@@ -0,0 +1,72 @@
package fetcher
import (
"context"
"encoding/json"
"errors"
"time"
)
type TriggerSource string
const (
TriggerScheduled TriggerSource = "scheduled"
TriggerManual TriggerSource = "manual"
)
var ErrJobSkipped = errors.New("protocol job skipped")
var ErrConfigChanged = errors.New("protocol configuration changed during execution")
// ConfigState embeds the common backend-owned state section in a protocol config.
// Public config PUTs always retain this section from the latest stored config.
type ConfigState[T any] struct {
State T `json:"state"`
}
// ProtocolHost is scoped to one manager. UpdateState runs against the latest
// config under Downloader's lock; callbacks must not perform network I/O.
type ProtocolHost struct {
Read func(any) error
UpdateState func(func(config, state json.RawMessage) (any, error)) error
Fetch func(context.Context, string, int64, func([]byte) error) ([]byte, error)
}
func ReadConfig[T any](host ProtocolHost) (*T, error) {
var cfg T
if err := host.Read(&cfg); err != nil {
return nil, err
}
return &cfg, nil
}
// UpdateState gives the callback a detached config snapshot and mutable state.
// Only the state is committed; edits to the config snapshot are discarded.
func UpdateState[C, S any](host ProtocolHost, update func(*C, *S) error) error {
return host.UpdateState(func(rawConfig, rawState json.RawMessage) (any, error) {
var cfg C
var state S
if err := json.Unmarshal(rawConfig, &cfg); err != nil {
return nil, err
}
if err := json.Unmarshal(rawState, &state); err != nil {
return nil, err
}
if err := update(&cfg, &state); err != nil {
return nil, err
}
return &state, nil
})
}
type CronJob struct {
Name string
Every time.Duration
Timeout time.Duration
Run func(context.Context, TriggerSource) error
}
type CronProvider interface{ CronJobs(ProtocolHost) []CronJob }
// ConfigNormalizer is optional. It owns protocol validation and derived fields.
type ConfigNormalizer interface {
NormalizeConfig(json.RawMessage) (any, error)
}
+73
View File
@@ -0,0 +1,73 @@
# GitHub mirror execution
This package owns GitHub URL rewriting and execution strategies. It depends only
on the standard library and does not depend on Downloader, HTTP clients, Git, or
storage. Downloader persists `Config` at its top-level `githubMirror` field;
`pkg/base` preserves the existing configuration names through aliases.
## Race: resource reads
```go
body, err := githubmirror.Race(ctx, cfg.GithubMirror, rawURL,
githubmirror.Options[[]byte]{
Kind: githubmirror.Resource,
Timeout: 30 * time.Second,
},
func(ctx context.Context, candidate string) ([]byte, error) {
return fetchAndValidate(ctx, client, candidate)
},
)
```
The callback owns HTTP requests, size limits, and content validation. The first
completed valid result wins. Other operations are canceled and drained. There is
no HEAD-probe assumption or client-specific HTTP behavior inside the executor.
## Fallback: repository operations
```go
repo, err := githubmirror.Fallback(ctx, cfg.GithubMirror, officialURL,
githubmirror.Options[*git.Repository]{
Kind: githubmirror.Repository,
},
func(ctx context.Context, candidate string) (*git.Repository, error) {
// Use a clean temporary checkout for each attempt.
return clone(ctx, candidate)
},
)
```
The official URL is attempted first; mirrors follow their configured order.
jsDelivr is excluded for repositories. Stop on the first success. Perform
persistent installation or other business state changes after success, rather
than inside a potentially repeated attempt.
## Results with resources
For temporary files, set `Options[*Download].Discard` to remove the file. Return
the partial `*Download` alongside errors, so cleanup also covers failed reads or
validation. The executor discards every failure and every race loser exactly
once. The caller owns and must track/release the winning result.
- `Timeout == 0` uses 30 seconds per resource attempt and no overall deadline
for repository attempts; a negative timeout is invalid. Extension Git HTTP
transport limits connections/TLS handshakes to 10 seconds and resets a 60-second
read deadline as data arrives. Slow continuous transfers can finish, while
stalled sources fail and allow ordered fallback. Parent cancellation still applies.
- Attempts must honor the supplied context; draining waits for them to return.
- Attempt contexts are canceled after return, so callbacks must finish reading
HTTP bodies before returning. Returning a still-streaming response is unsupported.
- Discard callbacks must accept zero values, must not panic, and run serially.
- Parent cancellation wins over a result selected while the parent is canceled;
the selected result is then discarded as well.
- Errors preserve their causes through wrapping/joining for `errors.Is`.
- `github.com` and `raw.githubusercontent.com` expand into eligible mirrors.
jsDelivr is only eligible for raw source resources. `api.github.com` is never
mirrored and is rejected by the public mirror candidates API.
- Client application update checks request the official GitHub API directly,
falling back to the independent Gopeed website API on request or validation
failure. Only package downloads obtain URLs from
`POST /api/v1/mirrors/github/candidates`. Candidate generation never performs
HTTP requests.
- Disabled mirrors and other URLs execute only the original URL.
- Config is a caller-owned immutable snapshot for the duration of a call.
+90
View File
@@ -0,0 +1,90 @@
// Package githubmirror rewrites GitHub URLs and executes mirrored operations.
package githubmirror
import (
"fmt"
"net/url"
"strings"
)
type Mirror struct {
Type string `json:"type"`
URL string `json:"url"`
IsBuiltIn bool `json:"isBuiltIn"`
IsDeleted bool `json:"isDeleted"`
}
type Config struct {
Enabled bool `json:"enabled"`
Mirrors []Mirror `json:"mirrors"`
}
func DefaultConfig() *Config {
return &Config{Enabled: true, Mirrors: []Mirror{
{Type: "jsdelivr", URL: "https://fastly.jsdelivr.net/gh", IsBuiltIn: true},
{Type: "ghProxy", URL: "https://fastgit.cc", IsBuiltIn: true},
{Type: "ghProxy", URL: "https://gh-proxy.com", IsBuiltIn: true},
{Type: "ghProxy", URL: "https://ghfast.top", IsBuiltIn: true},
}}
}
func (c *Config) Validate() error {
if len(c.Mirrors) > 32 {
return fmt.Errorf("too many GitHub mirrors")
}
for _, mirror := range c.Mirrors {
u, err := url.Parse(mirror.URL)
if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
return fmt.Errorf("invalid GitHub mirror URL")
}
if mirror.Type != "jsdelivr" && mirror.Type != "ghProxy" {
return fmt.Errorf("invalid GitHub mirror type: %s", mirror.Type)
}
}
return nil
}
// Kind describes what a candidate endpoint must support.
type Kind uint8
const (
Resource Kind = iota
Repository
)
// Candidates preserves the official URL first. jsDelivr is only eligible for
// raw source resources; Git operations use mirrors supporting Git Smart HTTP.
// GitHub API URLs are never mirrored.
func Candidates(c *Config, raw string, kind Kind) []string {
result := []string{raw}
u, err := url.Parse(raw)
if err != nil || c == nil || !c.Enabled || u.User != nil || (u.Scheme != "https" && u.Scheme != "http") {
return result
}
if u.Host != "github.com" && u.Host != "raw.githubusercontent.com" {
return result
}
seen := map[string]bool{raw: true}
for _, mirror := range c.Mirrors {
if mirror.IsDeleted {
continue
}
var candidate string
switch mirror.Type {
case "ghProxy":
candidate = strings.TrimRight(mirror.URL, "/") + "/" + raw
case "jsdelivr":
if kind == Repository || u.Host != "raw.githubusercontent.com" {
continue
}
parts := strings.SplitN(strings.TrimPrefix(u.EscapedPath(), "/"), "/", 4)
if len(parts) != 4 {
continue
}
candidate = strings.TrimRight(mirror.URL, "/") + "/" + parts[0] + "/" + parts[1] + "@" + parts[2] + "/" + parts[3]
}
if candidate != "" && !seen[candidate] {
result = append(result, candidate)
seen[candidate] = true
}
}
return result
}
+38
View File
@@ -0,0 +1,38 @@
package githubmirror
import (
"reflect"
"testing"
)
func TestMirrorCandidates(t *testing.T) {
cfg := &Config{Enabled: true, Mirrors: []Mirror{
{Type: "jsdelivr", URL: "https://cdn.example/gh/"},
{Type: "ghProxy", URL: "https://mirror.example/"},
{Type: "ghProxy", URL: "https://deleted.example", IsDeleted: true},
}}
source := "https://raw.githubusercontent.com/owner/repo/main/a.txt"
want := []string{source, "https://cdn.example/gh/owner/repo@main/a.txt", "https://mirror.example/" + source}
if got := Candidates(cfg, source, Resource); !reflect.DeepEqual(got, want) {
t.Fatalf("got %v", got)
}
repo := "https://github.com/owner/repo.git"
if got := Candidates(cfg, repo, Repository); !reflect.DeepEqual(got, []string{repo, "https://mirror.example/" + repo}) {
t.Fatalf("git candidates %v", got)
}
for _, raw := range []string{"https://github.com.evil.example/a", "https://example.com/a", "https://user:[email protected]/a"} {
if got := Candidates(cfg, raw, Resource); !reflect.DeepEqual(got, []string{raw}) {
t.Fatalf("unexpected mirror: %v", got)
}
}
apiURL := "https://api.github.com/repos/GopeedLab/gopeed/releases?per_page=10"
for _, kind := range []Kind{Resource, Repository} {
if got := Candidates(cfg, apiURL, kind); !reflect.DeepEqual(got, []string{apiURL}) {
t.Fatalf("GitHub API must never use mirrors: %v", got)
}
}
cfg.Enabled = false
if got := Candidates(cfg, source, Resource); !reflect.DeepEqual(got, []string{source}) {
t.Fatalf("disabled mirror: %v", got)
}
}
+160
View File
@@ -0,0 +1,160 @@
package githubmirror
import (
"context"
"errors"
"fmt"
"time"
)
// Options configures execution, not URL persistence. Discard releases results
// from failed attempts and successful race losers, including partial results
// returned together with an error. The caller owns the winning result.
type Options[T any] struct {
Kind Kind
// Timeout limits each attempt. Zero uses 30 seconds for resources and no
// overall deadline for repositories, whose transport owns idle timeouts.
Timeout time.Duration
Discard func(T)
}
// Attempt must honor ctx and return a completed, validated result. Its context
// is canceled after it returns, so live response streams must not be returned.
// Return any partially acquired resources along with errors so Discard can
// release them. Failed attempts must not mutate persistent business state.
type Attempt[T any] func(context.Context, string) (T, error)
const defaultTimeout = 30 * time.Second
func prepare[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) ([]string, time.Duration, error) {
if err := ctx.Err(); err != nil {
return nil, 0, err
}
if raw == "" {
return nil, 0, errors.New("empty GitHub source URL")
}
if options.Kind != Resource && options.Kind != Repository {
return nil, 0, errors.New("invalid GitHub resource kind")
}
if options.Timeout < 0 {
return nil, 0, errors.New("invalid GitHub attempt timeout")
}
if attempt == nil {
return nil, 0, errors.New("GitHub mirror attempt is required")
}
if config != nil {
if err := config.Validate(); err != nil {
return nil, 0, err
}
}
timeout := options.Timeout
if timeout == 0 && options.Kind == Resource {
timeout = defaultTimeout
}
return Candidates(config, raw, options.Kind), timeout, nil
}
func invoke[T any](ctx context.Context, timeout time.Duration, candidate string, attempt Attempt[T]) (value T, err error) {
var cancel context.CancelFunc
if timeout > 0 {
ctx, cancel = context.WithTimeout(ctx, timeout)
} else {
ctx, cancel = context.WithCancel(ctx)
}
defer cancel()
defer func() {
if recovered := recover(); recovered != nil {
err = fmt.Errorf("GitHub mirror attempt panicked: %v", recovered)
}
}()
value, err = attempt(ctx, candidate)
if err == nil {
err = ctx.Err()
}
return
}
func discard[T any](options Options[T], value T) {
if options.Discard != nil {
options.Discard(value)
}
}
// Race executes the official URL and eligible mirrors concurrently. The first
// completed successful result wins; losing attempts are canceled and drained
// before return so no temporary files or other resources outlive the call.
// HTTP status/content validation is the attempt's responsibility.
func Race[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) (T, error) {
var zero T
candidates, timeout, err := prepare(ctx, config, raw, options, attempt)
if err != nil {
return zero, err
}
raceCtx, cancel := context.WithCancel(ctx)
defer cancel()
type outcome struct {
index int
value T
err error
}
outcomes := make(chan outcome, len(candidates))
for index, candidate := range candidates {
go func(index int, candidate string) {
value, err := invoke(raceCtx, timeout, candidate, attempt)
outcomes <- outcome{index, value, err}
}(index, candidate)
}
var winner T
won := false
failures := make([]error, len(candidates))
for range candidates {
outcome := <-outcomes
if !won && outcome.err == nil && ctx.Err() == nil {
winner = outcome.value
won = true
cancel()
} else {
discard(options, outcome.value)
if outcome.err != nil {
failures[outcome.index] = fmt.Errorf("GitHub source %d: %w", outcome.index+1, outcome.err)
}
if ctx.Err() != nil {
cancel()
}
}
}
if err := ctx.Err(); err != nil {
if won {
discard(options, winner)
}
return zero, err
}
if won {
return winner, nil
}
return zero, fmt.Errorf("all GitHub sources failed: %w", errors.Join(failures...))
}
// Fallback executes the official URL first, then eligible mirrors in their
// configured order. It stops immediately on success and cleans every failure.
func Fallback[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) (T, error) {
var zero T
candidates, timeout, err := prepare(ctx, config, raw, options, attempt)
if err != nil {
return zero, err
}
failures := make([]error, 0, len(candidates))
for index, candidate := range candidates {
if err := ctx.Err(); err != nil {
return zero, err
}
value, err := invoke(ctx, timeout, candidate, attempt)
if err == nil && ctx.Err() == nil {
return value, nil
}
discard(options, value)
if parentErr := ctx.Err(); parentErr != nil {
return zero, parentErr
}
failures = append(failures, fmt.Errorf("GitHub source %d: %w", index+1, err))
}
return zero, fmt.Errorf("all GitHub sources failed: %w", errors.Join(failures...))
}
+214
View File
@@ -0,0 +1,214 @@
package githubmirror
import (
"context"
"errors"
"reflect"
"strings"
"sync"
"testing"
"time"
)
const testSource = "https://github.com/owner/repo.git"
func testConfig() *Config {
return &Config{Enabled: true, Mirrors: []Mirror{
{Type: "ghProxy", URL: "https://first.example"},
{Type: "ghProxy", URL: "https://second.example"},
}}
}
func TestRaceWinsAndDrainsOwnedResults(t *testing.T) {
config := testConfig()
candidates := Candidates(config, testSource, Resource)
var started sync.WaitGroup
started.Add(len(candidates))
discarded := []string{}
winner, err := Race(context.Background(), config, testSource, Options[string]{
Discard: func(value string) { discarded = append(discarded, value) },
}, func(ctx context.Context, address string) (string, error) {
started.Done()
started.Wait()
switch address {
case candidates[0]:
return address, errors.New("official offline")
case candidates[1]:
return address, nil
default:
<-ctx.Done()
return address, ctx.Err()
}
})
if err != nil || winner != candidates[1] {
t.Fatalf("winner=%s err=%v", winner, err)
}
if len(discarded) != 2 || !contains(discarded, candidates[0]) || !contains(discarded, candidates[2]) {
t.Fatalf("discarded %v", discarded)
}
}
func contains(values []string, value string) bool {
for _, item := range values {
if item == value {
return true
}
}
return false
}
func TestRaceCleansSimultaneousSuccessfulLosers(t *testing.T) {
config := testConfig()
var started sync.WaitGroup
started.Add(3)
cleaned := []string{}
winner, err := Race(context.Background(), config, testSource, Options[string]{Discard: func(value string) { cleaned = append(cleaned, value) }}, func(_ context.Context, address string) (string, error) {
started.Done()
started.Wait()
return address, nil
})
if err != nil || winner == "" || len(cleaned) != 2 || contains(cleaned, winner) {
t.Fatalf("winner=%s discarded=%v err=%v", winner, cleaned, err)
}
}
func TestRaceParentCancellationCleansWinnerToo(t *testing.T) {
config := testConfig()
config.Mirrors = config.Mirrors[:1]
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cleaned := []string{}
winner, err := Race(ctx, config, testSource, Options[string]{Discard: func(value string) { cleaned = append(cleaned, value) }}, func(ctx context.Context, address string) (string, error) {
if address == testSource {
return address, nil
}
<-ctx.Done()
cancel()
return address, ctx.Err()
})
if !errors.Is(err, context.Canceled) || winner != "" || len(cleaned) != 2 || !contains(cleaned, testSource) {
t.Fatalf("winner=%s cleanup=%v err=%v", winner, cleaned, err)
}
}
func TestRaceTimeoutAndErrorAggregation(t *testing.T) {
expected := errors.New("offline")
config := testConfig()
config.Mirrors = config.Mirrors[:1]
discarded := 0
_, err := Race(context.Background(), config, testSource, Options[string]{Timeout: 10 * time.Millisecond, Discard: func(string) { discarded++ }}, func(ctx context.Context, address string) (string, error) {
if address == testSource {
return address, expected
}
<-ctx.Done()
return address, ctx.Err()
})
if !errors.Is(err, expected) || !errors.Is(err, context.DeadlineExceeded) || discarded != 2 {
t.Fatalf("discarded=%d err=%v", discarded, err)
}
}
func TestFallbackPreservesOrderAndStopsOnSuccess(t *testing.T) {
config := testConfig()
candidates := Candidates(config, testSource, Repository)
tried := []string{}
cleaned := []string{}
winner, err := Fallback(context.Background(), config, testSource, Options[string]{Kind: Repository, Discard: func(value string) { cleaned = append(cleaned, value) }}, func(_ context.Context, address string) (string, error) {
tried = append(tried, address)
if address == candidates[1] {
return address, nil
}
return address, errors.New("offline")
})
if err != nil || winner != candidates[1] || !reflect.DeepEqual(tried, candidates[:2]) || !reflect.DeepEqual(cleaned, candidates[:1]) {
t.Fatalf("winner=%s tried=%v discarded=%v err=%v", winner, tried, cleaned, err)
}
tried = nil
_, err = Fallback(context.Background(), config, testSource, Options[string]{Kind: Repository}, func(_ context.Context, address string) (string, error) {
tried = append(tried, address)
return address, nil
})
if err != nil || !reflect.DeepEqual(tried, candidates[:1]) {
t.Fatalf("tried=%v err=%v", tried, err)
}
}
func TestFallbackTimeoutContinuesWithFreshContext(t *testing.T) {
discarded := []string{}
winner, err := Fallback(context.Background(), testConfig(), testSource, Options[string]{Kind: Repository, Timeout: 10 * time.Millisecond, Discard: func(value string) { discarded = append(discarded, value) }}, func(ctx context.Context, address string) (string, error) {
if address == testSource {
<-ctx.Done()
return address, ctx.Err()
}
if err := ctx.Err(); err != nil {
return address, err
}
return address, nil
})
if err != nil || !strings.HasPrefix(winner, "https://first.example/") || !reflect.DeepEqual(discarded, []string{testSource}) {
t.Fatalf("winner=%s cleanup=%v err=%v", winner, discarded, err)
}
}
func TestFallbackCancellationStopsBeforeNextMirror(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
calls := 0
cleaned := 0
_, err := Fallback(ctx, testConfig(), testSource, Options[string]{Kind: Repository, Discard: func(string) { cleaned++ }}, func(ctx context.Context, address string) (string, error) {
calls++
cancel()
return address, ctx.Err()
})
if !errors.Is(err, context.Canceled) || calls != 1 || cleaned != 1 {
t.Fatalf("calls=%d discarded=%d err=%v", calls, cleaned, err)
}
}
func TestExecutorsDoNotStartInvalidOrCanceledRequests(t *testing.T) {
canceled, cancel := context.WithCancel(context.Background())
cancel()
for _, test := range []struct {
ctx context.Context
config *Config
options Options[string]
attempt Attempt[string]
}{
{ctx: canceled, options: Options[string]{}, attempt: func(context.Context, string) (string, error) { t.Fatal("called after cancellation"); return "", nil }},
{ctx: context.Background(), options: Options[string]{Kind: 99}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid kind"); return "", nil }},
{ctx: context.Background(), options: Options[string]{Timeout: -time.Second}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid timeout"); return "", nil }},
{ctx: context.Background(), options: Options[string]{}, attempt: nil},
{ctx: context.Background(), config: &Config{Mirrors: []Mirror{{Type: "invalid", URL: "https://mirror.example"}}}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid config"); return "", nil }},
} {
if _, err := Race(test.ctx, test.config, testSource, test.options, test.attempt); err == nil {
t.Fatal("invalid race accepted")
}
if _, err := Fallback(test.ctx, test.config, testSource, test.options, test.attempt); err == nil {
t.Fatal("invalid fallback accepted")
}
}
}
func TestAttemptPanicBecomesError(t *testing.T) {
attempt := func(context.Context, string) (string, error) { panic("broken operation") }
if _, err := Race(context.Background(), nil, testSource, Options[string]{}, attempt); err == nil || !strings.Contains(err.Error(), "broken operation") {
t.Fatal(err)
}
if _, err := Fallback(context.Background(), nil, testSource, Options[string]{}, attempt); err == nil || !strings.Contains(err.Error(), "broken operation") {
t.Fatal(err)
}
}
func TestNonGithubResourcesAndDisabledMirrorsUseOnlyOriginal(t *testing.T) {
for _, test := range []struct {
config *Config
raw string
}{{testConfig(), "https://example.com/resource"}, {&Config{Enabled: false, Mirrors: testConfig().Mirrors}, testSource}} {
calls := 0
got, err := Race(context.Background(), test.config, test.raw, Options[string]{}, func(_ context.Context, address string) (string, error) { calls++; return address, nil })
if err != nil || got != test.raw || calls != 1 {
t.Fatalf("got=%s calls=%d err=%v", got, calls, err)
}
}
}
func TestRepositoryAttemptsHaveNoDefaultDeadline(t *testing.T) {
value, err := Fallback(context.Background(), nil, testSource, Options[string]{Kind: Repository}, func(ctx context.Context, raw string) (string, error) {
if _, ok := ctx.Deadline(); ok {
t.Fatal("repository attempt has a total deadline")
}
return raw, nil
})
if err != nil || value != testSource {
t.Fatalf("value=%s error=%v", value, err)
}
}
+23 -2
View File
@@ -1,8 +1,29 @@
package bt
import (
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
)
type trackerSubscriptionStatus struct {
Success bool `json:"success"`
TrackerCount int `json:"trackerCount"`
LastFetchTime time.Time `json:"lastFetchTime"`
}
type state struct {
TrackerSubscriptionStatuses map[string]trackerSubscriptionStatus `json:"trackerSubscriptionStatuses"`
TrackerSources map[string][]string `json:"trackerSources"`
LastTrackerUpdateTime time.Time `json:"lastTrackerUpdateTime"`
}
type config struct {
ListenPort int `json:"listenPort"`
Trackers []string `json:"trackers"`
fetcher.ConfigState[state]
TrackerSubscribeURLs []string `json:"trackerSubscribeUrls"`
AutoUpdateTrackers bool `json:"autoUpdateTrackers"`
CustomTrackers []string `json:"customTrackers"`
ListenPort int `json:"listenPort"`
// SeedKeep is always keep seeding after downloading is complete, unless manually stopped.
SeedKeep bool `json:"seedKeep"`
// SeedRatio is the ratio of uploaded data to downloaded data to seed.
+18 -2
View File
@@ -198,7 +198,7 @@ func TestBTDownloadSources(t *testing.T) {
}
func TestBTDirectDownloadWithExtraTrackers(t *testing.T) {
for _, source := range []string{"request", "config"} {
for _, source := range []string{"request", "config", "subscription"} {
t.Run(source, func(t *testing.T) {
fixture := makeBTFixture(t, map[string][]byte{
"a.bin": btPayload(20*1024, "direct-a"),
@@ -217,7 +217,23 @@ func TestBTDirectDownloadWithExtraTrackers(t *testing.T) {
if source == "request" {
req.Extra = &btprotocol.ReqExtra{Trackers: []string{fixture.TrackerURL}}
} else {
f.config.Trackers = []string{fixture.TrackerURL}
backendConfig := new(FetcherManager).DefaultConfig().(*config)
if source == "config" {
backendConfig.CustomTrackers = []string{fixture.TrackerURL}
} else {
backendConfig.TrackerSubscribeURLs = []string{"https://subscription.example/list"}
backendConfig.State.TrackerSources[backendConfig.TrackerSubscribeURLs[0]] = []string{fixture.TrackerURL}
}
f.ctl.GetConfig = func(dst any) {
raw, err := json.Marshal(backendConfig)
if err != nil {
t.Error(err)
return
}
if err := json.Unmarshal(raw, dst); err != nil {
t.Error(err)
}
}
}
// CreateDirect supplies options before Downloader resolves the task.
dir := filepath.Join(t.TempDir(), "direct-download")
+10 -5
View File
@@ -744,6 +744,7 @@ func (f *Fetcher) WaitUpload() (err error) {
}
func (f *Fetcher) addTorrent(req *base.Request) (err error) {
f.ctl.GetConfig(&f.config)
if err = base.ParseReqExtra[bt.ReqExtra](req); err != nil {
return
}
@@ -816,10 +817,8 @@ func (f *Fetcher) addTorrent(req *base.Request) (err error) {
}
}
}
if len(f.config.Trackers) > 0 {
for _, tracker := range f.config.Trackers {
trackers[tracker] = true
}
for _, tracker := range f.config.effectiveTrackers() {
trackers[tracker] = true
}
if len(trackers) > 0 {
for tracker := range trackers {
@@ -947,8 +946,14 @@ func (fm *FetcherManager) AutoRename() bool {
func (fm *FetcherManager) DefaultConfig() any {
return &config{
TrackerSubscribeURLs: append([]string{}, defaultTrackerSubscribeURLs...),
AutoUpdateTrackers: true,
CustomTrackers: []string{},
ConfigState: fetcher.ConfigState[state]{State: state{
TrackerSources: map[string][]string{},
TrackerSubscriptionStatuses: map[string]trackerSubscriptionStatus{},
}},
ListenPort: 0,
Trackers: []string{},
SeedKeep: false,
SeedRatio: 1.0,
SeedTime: 120 * 60,
+1 -1
View File
@@ -227,7 +227,7 @@ func buildConfigFetcher(t testing.TB, proxyConfig *base.DownloaderProxyConfig) f
fetcher := new(FetcherManager).Build()
newController := controller.NewController()
mockCfg := config{
Trackers: []string{
CustomTrackers: []string{
"udp://tracker.birkenwald.de:6969/announce",
"udp://tracker.bitsearch.to:1337/announce",
}}
+234
View File
@@ -0,0 +1,234 @@
package bt
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/url"
"slices"
"strings"
"sync"
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
)
var defaultTrackerSubscribeURLs = []string{
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_https.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_udp.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ws.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best.txt",
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best_ip.txt",
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/all.txt",
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best.txt",
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/http.txt",
}
func validTracker(raw string) bool {
u, err := url.Parse(raw)
if err != nil || u.Hostname() == "" || u.User != nil {
return false
}
switch u.Scheme {
case "udp", "http", "https", "ws", "wss":
return true
}
return false
}
func uniqueTrackers(groups ...[]string) []string {
result := []string{}
seen := map[string]bool{}
for _, group := range groups {
for _, raw := range group {
value := strings.TrimSpace(raw)
if value != "" && !seen[value] {
seen[value] = true
result = append(result, value)
}
}
}
return result
}
// effectiveTrackers derives the client list without persisting duplicate lists.
func (c *config) effectiveTrackers() []string {
groups := [][]string{c.CustomTrackers}
for _, source := range c.TrackerSubscribeURLs {
groups = append(groups, c.State.TrackerSources[source])
}
return uniqueTrackers(groups...)
}
func (fm *FetcherManager) NormalizeConfig(raw json.RawMessage) (any, error) {
c := fm.DefaultConfig().(*config)
if err := json.Unmarshal(raw, c); err != nil {
return nil, err
}
if c.ListenPort < 0 || c.ListenPort > 65535 || c.SeedRatio < 0 || c.SeedTime < 0 {
return nil, errors.New("invalid BT configuration")
}
if len(c.TrackerSubscribeURLs) > 64 {
return nil, errors.New("too many tracker subscriptions")
}
c.TrackerSubscribeURLs = uniqueTrackers(c.TrackerSubscribeURLs)
for _, source := range c.TrackerSubscribeURLs {
u, err := url.Parse(source)
if err != nil || u == nil || (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil {
return nil, errors.New("invalid tracker subscription URL")
}
}
c.CustomTrackers = uniqueTrackers(c.CustomTrackers)
for _, tracker := range c.CustomTrackers {
if !validTracker(tracker) {
return nil, fmt.Errorf("invalid tracker URL: %s", tracker)
}
}
if c.State.TrackerSources == nil {
c.State.TrackerSources = map[string][]string{}
}
for source := range c.State.TrackerSources {
if !slices.Contains(c.TrackerSubscribeURLs, source) {
delete(c.State.TrackerSources, source)
}
}
if c.State.TrackerSubscriptionStatuses == nil {
c.State.TrackerSubscriptionStatuses = map[string]trackerSubscriptionStatus{}
}
for source := range c.State.TrackerSubscriptionStatuses {
if !slices.Contains(c.TrackerSubscribeURLs, source) {
delete(c.State.TrackerSubscriptionStatuses, source)
}
}
return c, nil
}
func (fm *FetcherManager) CronJobs(host fetcher.ProtocolHost) []fetcher.CronJob {
return []fetcher.CronJob{{
Name: "update-trackers",
Every: 24 * time.Hour,
Timeout: 90 * time.Second,
Run: func(ctx context.Context, source fetcher.TriggerSource) error {
return fm.updateTrackers(ctx, host, source)
},
}}
}
func parseTrackers(data []byte) ([]string, error) {
scanner := bufio.NewScanner(bytes.NewReader(data))
scanner.Buffer(make([]byte, 4096), 64*1024)
result := []string{}
for scanner.Scan() {
line := strings.TrimSpace(strings.TrimPrefix(scanner.Text(), "\ufeff"))
if validTracker(line) {
result = append(result, line)
}
if len(result) > 20000 {
return nil, errors.New("too many trackers")
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
if len(result) == 0 {
return nil, errors.New("subscription contains no valid trackers")
}
return uniqueTrackers(result), nil
}
func (fm *FetcherManager) updateTrackers(ctx context.Context, host fetcher.ProtocolHost, source fetcher.TriggerSource) error {
snapshot, err := fetcher.ReadConfig[config](host)
if err != nil {
return err
}
if (source == fetcher.TriggerScheduled && !snapshot.AutoUpdateTrackers) || len(snapshot.TrackerSubscribeURLs) == 0 {
return fetcher.ErrJobSkipped
}
fetched := map[string][]string{}
statuses := map[string]trackerSubscriptionStatus{}
var failures []error
var mu sync.Mutex
var wg sync.WaitGroup
slots := make(chan struct{}, 4)
for _, address := range snapshot.TrackerSubscribeURLs {
wg.Add(1)
go func(address string) {
defer wg.Done()
select {
case slots <- struct{}{}:
defer func() { <-slots }()
case <-ctx.Done():
return
}
data, err := host.Fetch(ctx, address, 2<<20, func(body []byte) error { _, err := parseTrackers(body); return err })
var trackers []string
if err == nil {
trackers, err = parseTrackers(data)
}
mu.Lock()
defer mu.Unlock()
statuses[address] = trackerSubscriptionStatus{Success: err == nil, TrackerCount: len(trackers), LastFetchTime: time.Now().UTC()}
if err != nil {
failures = append(failures, fmt.Errorf("subscription %s: %w", address, err))
} else {
fetched[address] = trackers
}
}(address)
}
wg.Wait()
if err := ctx.Err(); err != nil {
failures = append(failures, err)
}
if len(statuses) == 0 {
return errors.Join(failures...)
}
err = fetcher.UpdateState[config, state](host, func(current *config, state *state) error {
if !slices.Equal(current.TrackerSubscribeURLs, snapshot.TrackerSubscribeURLs) {
return fetcher.ErrConfigChanged
}
if state.TrackerSources == nil {
state.TrackerSources = map[string][]string{}
}
if state.TrackerSubscriptionStatuses == nil {
state.TrackerSubscriptionStatuses = map[string]trackerSubscriptionStatus{}
}
for address, status := range statuses {
state.TrackerSubscriptionStatuses[address] = status
}
for address, trackers := range fetched {
state.TrackerSources[address] = trackers
}
if len(failures) == 0 {
state.LastTrackerUpdateTime = time.Now().UTC()
}
return nil
})
if err != nil {
return err
}
current, err := fetcher.ReadConfig[config](host)
if err != nil {
return err
}
fm.applyTrackers(current.effectiveTrackers())
return errors.Join(failures...)
}
func (fm *FetcherManager) applyTrackers(trackers []string) {
lock.Lock()
defer lock.Unlock()
if client == nil {
return
}
announce := make([][]string, 0, len(trackers))
for _, tracker := range trackers {
announce = append(announce, []string{tracker})
}
for _, t := range client.Torrents() {
info := t.Info()
if info == nil || (info.Private != nil && *info.Private) {
continue
}
t.AddTrackers(announce)
}
}
+217
View File
@@ -0,0 +1,217 @@
package bt
import (
"context"
"encoding/json"
"errors"
"reflect"
"sync"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
)
func trackerTestHost(t *testing.T, c *config, fetch func(context.Context, string, int64, func([]byte) error) ([]byte, error)) (fetcher.ProtocolHost, *sync.Mutex) {
t.Helper()
mu := &sync.Mutex{}
return fetcher.ProtocolHost{
Read: func(dst any) error {
mu.Lock()
defer mu.Unlock()
raw, err := json.Marshal(c)
if err != nil {
return err
}
return json.Unmarshal(raw, dst)
},
UpdateState: func(change func(config, state json.RawMessage) (any, error)) error {
mu.Lock()
defer mu.Unlock()
raw, _ := json.Marshal(c)
currentState, _ := json.Marshal(c.State)
next, err := change(raw, currentState)
if err != nil {
return err
}
raw, err = json.Marshal(next)
if err != nil {
return err
}
return json.Unmarshal(raw, &c.State)
},
Fetch: fetch,
}, mu
}
func TestTrackerRefreshPartialSuccessUsesLatestConfig(t *testing.T) {
fm := &FetcherManager{}
cfg := fm.DefaultConfig().(*config)
cfg.TrackerSubscribeURLs = []string{"https://a.example/list", "https://b.example/list"}
cfg.CustomTrackers = []string{"udp://custom.example:80/announce"}
cfg.State.TrackerSources = map[string][]string{cfg.TrackerSubscribeURLs[0]: {"udp://old.example:80/announce"}, cfg.TrackerSubscribeURLs[1]: {"udp://cached.example:80/announce"}}
host, mu := trackerTestHost(t, cfg, nil)
host.Fetch = func(ctx context.Context, address string, _ int64, validate func([]byte) error) ([]byte, error) {
if address == "https://b.example/list" {
return nil, errors.New("offline")
}
// Simulate a frontend update while the network request is in flight.
mu.Lock()
cfg.CustomTrackers = []string{"udp://new-custom.example:80/announce"}
mu.Unlock()
data := []byte("udp://fresh.example:80/announce\n# ignored\nudp://fresh.example:80/announce\n")
return data, validate(data)
}
err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual)
if err == nil {
t.Fatal("partial failure must be reported")
}
want := []string{"udp://new-custom.example:80/announce", "udp://fresh.example:80/announce", "udp://cached.example:80/announce"}
if !reflect.DeepEqual(cfg.effectiveTrackers(), want) {
t.Fatalf("got %v", cfg.effectiveTrackers())
}
success := cfg.State.TrackerSubscriptionStatuses["https://a.example/list"]
failure := cfg.State.TrackerSubscriptionStatuses["https://b.example/list"]
if !success.Success || success.TrackerCount != 1 || success.LastFetchTime.IsZero() || failure.Success || failure.TrackerCount != 0 || failure.LastFetchTime.IsZero() {
t.Fatalf("unexpected subscription statuses: %+v", cfg.State.TrackerSubscriptionStatuses)
}
if !cfg.State.LastTrackerUpdateTime.IsZero() {
t.Fatal("partial refresh marked fully successful")
}
}
func TestTrackerRefreshDiscardsChangedSubscriptions(t *testing.T) {
fm := &FetcherManager{}
cfg := fm.DefaultConfig().(*config)
cfg.TrackerSubscribeURLs = []string{"https://a.example/list"}
host, mu := trackerTestHost(t, cfg, nil)
host.Fetch = func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
mu.Lock()
cfg.TrackerSubscribeURLs = []string{}
mu.Unlock()
return []byte("udp://fresh.example:80/announce"), nil
}
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); !errors.Is(err, fetcher.ErrConfigChanged) {
t.Fatal(err)
}
if len(cfg.State.TrackerSources) != 0 {
t.Fatal("outdated source was committed")
}
}
func TestTrackerRefreshManualWorksWhenAutomaticDisabled(t *testing.T) {
fm := &FetcherManager{}
cfg := fm.DefaultConfig().(*config)
cfg.AutoUpdateTrackers = false
cfg.TrackerSubscribeURLs = []string{"https://a.example/list"}
calls := 0
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
calls++
return []byte("udp://fresh.example:80/announce"), nil
})
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerScheduled); !errors.Is(err, fetcher.ErrJobSkipped) {
t.Fatal(err)
}
if calls != 0 {
t.Fatal("scheduled job fetched while disabled")
}
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); err != nil {
t.Fatal(err)
}
if calls != 1 || cfg.State.LastTrackerUpdateTime.IsZero() {
t.Fatal("manual refresh did not succeed")
}
}
func TestTrackerConfigPreservesExplicitSettings(t *testing.T) {
value, err := (&FetcherManager{}).NormalizeConfig(json.RawMessage(`{
"listenPort":6881, "trackerSubscribeUrls":[], "autoUpdateTrackers":false,
"customTrackers":["udp://custom.example:80/announce"]
}`))
if err != nil {
t.Fatal(err)
}
cfg := value.(*config)
if cfg.AutoUpdateTrackers || len(cfg.TrackerSubscribeURLs) != 0 || cfg.ListenPort != 6881 {
t.Fatalf("%+v", cfg)
}
if !reflect.DeepEqual(cfg.effectiveTrackers(), cfg.CustomTrackers) {
t.Fatalf("derived trackers: %v", cfg.effectiveTrackers())
}
}
func TestParseTrackerList(t *testing.T) {
got, err := parseTrackers([]byte("\ufeffudp://tracker.example:80/announce\r\n\n<html>error</html>\nhttps://tracker.example/announce\n"))
if err != nil || len(got) != 2 {
t.Fatalf("%v %v", got, err)
}
if _, err := parseTrackers([]byte("<html>mirror error</html>")); err == nil {
t.Fatal("invalid mirror body accepted")
}
}
func TestTrackerDefaultsRegisterDailyJob(t *testing.T) {
jobs := (&FetcherManager{}).CronJobs(fetcher.ProtocolHost{})
if len(jobs) != 1 || jobs[0].Name != "update-trackers" || jobs[0].Every != 24*time.Hour {
t.Fatal(jobs)
}
}
func TestTrackerRefreshAllFailuresPersistStatusesAndKeepCache(t *testing.T) {
fm := &FetcherManager{}
cfg := fm.DefaultConfig().(*config)
address := "https://custom.example/trackers.txt"
cfg.TrackerSubscribeURLs = []string{address}
cfg.State.TrackerSources[address] = []string{"udp://cached.example:80/announce"}
cfg.State.TrackerSubscriptionStatuses[address] = trackerSubscriptionStatus{Success: true, TrackerCount: 1, LastFetchTime: time.Now().Add(-time.Hour)}
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
return nil, errors.New("offline")
})
before := time.Now()
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); err == nil {
t.Fatal("failure was not reported")
}
status, ok := cfg.State.TrackerSubscriptionStatuses[address]
if !ok || status.Success || status.TrackerCount != 0 || status.LastFetchTime.Before(before) {
t.Fatalf("failed fetch status not stored: %+v", status)
}
if !reflect.DeepEqual(cfg.State.TrackerSources[address], []string{"udp://cached.example:80/announce"}) {
t.Fatal("last good cache lost")
}
if !reflect.DeepEqual(cfg.effectiveTrackers(), cfg.State.TrackerSources[address]) {
t.Fatal("cached trackers not retained")
}
}
func TestTrackerConfigPrunesRemovedSubscriptions(t *testing.T) {
fm := &FetcherManager{}
cfg := fm.DefaultConfig().(*config)
address := "https://custom.example/trackers.txt"
cfg.TrackerSubscribeURLs = []string{}
cfg.State.TrackerSources[address] = []string{"udp://cached.example:80/announce"}
cfg.State.TrackerSubscriptionStatuses[address] = trackerSubscriptionStatus{Success: true, TrackerCount: 1, LastFetchTime: time.Now()}
raw, _ := json.Marshal(cfg)
value, err := fm.NormalizeConfig(raw)
if err != nil {
t.Fatal(err)
}
next := value.(*config)
if len(next.State.TrackerSources) != 0 || len(next.State.TrackerSubscriptionStatuses) != 0 || len(next.effectiveTrackers()) != 0 {
t.Fatal("removed subscription still owns cached results")
}
}
func TestTrackerRefreshTimeoutPersistsCompletedAttemptStatus(t *testing.T) {
fm := &FetcherManager{}
cfg := fm.DefaultConfig().(*config)
address := "https://custom.example/trackers.txt"
cfg.TrackerSubscribeURLs = []string{address}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
cancel()
return nil, context.DeadlineExceeded
})
if err := fm.updateTrackers(ctx, host, fetcher.TriggerManual); !errors.Is(err, context.DeadlineExceeded) {
t.Fatal(err)
}
status, ok := cfg.State.TrackerSubscriptionStatuses[address]
if !ok || status.Success || status.TrackerCount != 0 || status.LastFetchTime.IsZero() {
t.Fatalf("timeout status lost: %+v", status)
}
}