Merge main into codex/tempdir-media-spooling

pkg/download/downloader.go

@@ -22,6 +22,7 @@ import (
 	"github.com/GopeedLab/gopeed/internal/fetcher"
 	"github.com/GopeedLab/gopeed/internal/logger"
 	"github.com/GopeedLab/gopeed/internal/tempfiles"
+	webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
 	"github.com/GopeedLab/gopeed/pkg/base"
 	"github.com/GopeedLab/gopeed/pkg/protocol/http"
 	"github.com/GopeedLab/gopeed/pkg/util"
@@ -98,9 +99,13 @@ type Progress struct {
 }
 
 type Downloader struct {
-	tempFiles       *tempfiles.Scope
-	Logger          *logger.Logger
-	ExtensionLogger *logger.Logger
+	tempFiles           *tempfiles.Scope
+	webviewProfilesLock sync.Mutex
+	webviewProfiles     map[string]*extensionWebViewProfile
+	webviewProxyLock    sync.Mutex
+	webviewProxy        *webviewproxy.Server
+	Logger              *logger.Logger
+	ExtensionLogger     *logger.Logger
 
 	cfg          *DownloaderConfig
 	fetcherCache map[string]fetcher.Fetcher
@@ -1144,6 +1149,11 @@ func (d *Downloader) doDelete(task *Task, force bool) (err error) {
 
 func (d *Downloader) Close() error {
 	d.closed.Store(true)
+	d.webviewProxyLock.Lock()
+	if d.webviewProxy != nil {
+		_ = d.webviewProxy.Close()
+	}
+	d.webviewProxyLock.Unlock()
 
 	closeArr := []func() error{
 		d.pauseAll,

pkg/download/engine/webview/rpc.go

@@ -5,6 +5,7 @@ import "encoding/json"
 const RPCEndpointPath = "/webview"
 
 const (
+	MethodProfileRemove     = "profile.remove"
 	MethodIsAvailable       = "webview.isAvailable"
 	MethodPageOpen          = "page.open"
 	MethodPageAddInitScript = "page.addInitScript"
@@ -68,6 +69,9 @@ type IsAvailableResult struct {
 }
 
 type PageOpenParams struct {
+	ProfileID string `json:"profileId,omitempty"`
+	DataPath  string `json:"dataPath,omitempty"`
+	ProxyURL  string `json:"proxyUrl,omitempty"`
 	Headless  bool   `json:"headless,omitempty"`
 	Debug     bool   `json:"debug,omitempty"`
 	Title     string `json:"title,omitempty"`
@@ -78,6 +82,9 @@ type PageOpenParams struct {
 
 func NewPageOpenParams(opts OpenOptions) PageOpenParams {
 	return PageOpenParams{
+		ProfileID: opts.ProfileID,
+		DataPath:  opts.DataPath,
+		ProxyURL:  opts.ProxyURL,
 		Headless:  opts.Headless,
 		Debug:     opts.Debug,
 		Title:     opts.Title,
@@ -141,3 +148,8 @@ type PageCloseParams struct {
 }
 
 type EmptyResult struct{}
+
+// ProfileRemoveParams belongs to the private host RPC.
+type ProfileRemoveParams struct {
+	ProfileID string `json:"profileId"`
+}

pkg/download/engine/webview/runtime.go

@@ -17,6 +17,11 @@ type Opener interface {
 	Open(opts OpenOptions) (Page, error)
 }
 
+// ProfileRemover is a host-only lifecycle operation, never exposed to extension JS.
+type ProfileRemover interface {
+	RemoveProfile(profileID, dataPath string) error
+}
+
 type Page interface {
 	AddInitScript(script string) error
 	Goto(url string, opts GotoOptions) error
@@ -29,6 +34,10 @@ type Page interface {
 }
 
 type OpenOptions struct {
+	// Host-owned fields are never parsed from extension JavaScript options.
+	ProfileID string
+	DataPath  string
+	ProxyURL  string
 	Headless  bool
 	Debug     bool
 	Title     string

pkg/download/engine/webview/runtime_test.go

@@ -336,3 +336,15 @@ func (p *fakePage) Close() error {
 	p.closed = true
 	return nil
 }
+
+func TestExtensionCannotSupplyHostProfileOrProxy(t *testing.T) {
+	opener := &fakeOpener{page: &fakePage{}}
+	runtime := NewRuntime(opener, true)
+	_, err := runtime.Open(map[string]any{"profileId": "other-extension", "dataPath": "/tmp/other-profile", "proxyUrl": "http://example.com:1234", "ProfileID": "other"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if opener.opts.ProfileID != "" || opener.opts.DataPath != "" || opener.opts.ProxyURL != "" {
+		t.Fatalf("extension controlled host settings: %+v", opener.opts)
+	}
+}

pkg/download/extension.go

@@ -64,6 +64,7 @@ func (d *Downloader) InstallExtensionByFolder(path string, devMode bool) (*Exten
 
 	// if extension is not installed, add it to the list, otherwise update it
 	installedExt := d.getExtension(ext.Identity)
+	newInstallation := installedExt == nil
 	if installedExt == nil {
 		ext.CreatedAt = time.Now()
 		ext.UpdatedAt = ext.CreatedAt
@@ -75,6 +76,11 @@ func (d *Downloader) InstallExtensionByFolder(path string, devMode bool) (*Exten
 	if err = d.storage.Put(bucketExtension, installedExt.Identity, installedExt); err != nil {
 		return nil, err
 	}
+	if newInstallation {
+		d.webviewProfilesLock.Lock()
+		delete(d.webviewProfiles, ext.Identity)
+		d.webviewProfilesLock.Unlock()
+	}
 	return installedExt, nil
 }
 
@@ -143,6 +149,9 @@ func (d *Downloader) DeleteExtension(identity string) error {
 	if err != nil {
 		return err
 	}
+	if err := d.removeExtensionWebViewProfile(identity); err != nil {
+		return fmt.Errorf("remove extension WebView data: %w", err)
+	}
 	// remove from disk
 	if !ext.DevMode {
 		if err := os.RemoveAll(d.ExtensionPath(ext)); err != nil {
@@ -367,7 +376,7 @@ func doTrigger[T any](d *Downloader, event ActivationEvent, req *base.Request, c
 					engine, session := d.newExtensionEngine()
 					defer session.CloseIfIdle()
 					gopeed.Runtime = &InstanceRuntime{
-						WebView: d.newExtensionWebViewRuntime(session),
+						WebView: d.newExtensionWebViewRuntime(session, ext.buildIdentity()),
 					}
 					err = injectGopeed(engine.Runtime, gopeed)
 					if err != nil {

pkg/download/extension_engine.go

@@ -30,7 +30,7 @@ func (d *Downloader) NewExtensionEngine(ext *Extension, settings map[string]any)
 			identity: ext.buildIdentity(),
 		},
 		Runtime: &InstanceRuntime{
-			WebView: d.newExtensionWebViewRuntime(session),
+			WebView: d.newExtensionWebViewRuntime(session, ext.buildIdentity()),
 		},
 	}
 	if err := injectGopeed(engine.Runtime, gopeed); err != nil {

pkg/download/extension_engine_session.go

@@ -137,13 +137,13 @@ func (d *Downloader) newExtensionEngine() (*engine.Engine, *engineSession) {
 	return e, session
 }
 
-func (d *Downloader) newExtensionWebViewRuntime(session *engineSession) *enginewebview.Runtime {
+func (d *Downloader) newExtensionWebViewRuntime(session *engineSession, identity string) *enginewebview.Runtime {
 	var (
 		opener    enginewebview.Opener
 		available bool
 	)
 	if provider := d.cfg.WebViewProvider; provider != nil && provider.IsAvailable() {
-		opener = provider
+		opener = &extensionWebViewOpener{downloader: d, provider: provider, identity: identity, profile: d.extensionWebViewProfile(identity)}
 		available = true
 	}
 	runtime := enginewebview.NewRuntime(opener, available)

pkg/download/extension_webview_config.go

@@ -0,0 +1,183 @@
+package download
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"sync"
+
+	"github.com/GopeedLab/gopeed/pkg/base"
+	"path/filepath"
+	"runtime"
+
+	webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
+	enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
+)
+
+// extensionWebViewOpener binds browser state to the owning extension. Neither
+// identity, filesystem paths nor proxy settings are accepted from JavaScript.
+type extensionWebViewOpener struct {
+	downloader *Downloader
+	provider   enginewebview.Provider
+	identity   string
+	profile    *extensionWebViewProfile
+}
+
+func (o *extensionWebViewOpener) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) {
+	if o.identity == "" {
+		return nil, fmt.Errorf("webview extension identity is required")
+	}
+	profile := o.profile
+	if profile == nil {
+		profile = o.downloader.extensionWebViewProfile(o.identity)
+	}
+	profile.mu.Lock()
+	defer profile.mu.Unlock()
+	if profile.revoked {
+		return nil, fmt.Errorf("extension WebView profile has been removed")
+	}
+	id, path, err := o.downloader.extensionWebViewLocation(o.identity)
+	if err != nil {
+		return nil, err
+	}
+	opts.ProfileID, opts.DataPath = id, path
+	opts.ProxyURL, err = o.downloader.webviewProxyURL()
+	if err != nil {
+		return nil, err
+	}
+	page, err := o.provider.Open(opts)
+	if err != nil {
+		return nil, err
+	}
+	tracked := &extensionWebViewPage{Page: page, profile: profile}
+	profile.pages[tracked] = struct{}{}
+	return tracked, nil
+}
+
+func (d *Downloader) webviewProxyURL() (string, error) {
+	d.webviewProxyLock.Lock()
+	defer d.webviewProxyLock.Unlock()
+	if d.closed.Load() {
+		return "", fmt.Errorf("downloader is closed")
+	}
+	if d.webviewProxy == nil {
+		server, err := webviewproxy.New(func(r *http.Request) (*url.URL, error) {
+			if d.blob != nil && d.blob.IsURL(r.URL.String()) {
+				return nil, nil
+			}
+			d.configLock.RLock()
+			var cfg base.DownloaderProxyConfig
+			if d.cfg.Proxy != nil {
+				cfg = *d.cfg.Proxy
+			}
+			d.configLock.RUnlock()
+			handler := cfg.ToHandler()
+			if handler == nil {
+				return nil, nil
+			}
+			return handler(r)
+		})
+		if err != nil {
+			return "", err
+		}
+		d.webviewProxy = server
+	}
+	if runtime.GOOS == "darwin" || runtime.GOOS == "ios" {
+		return d.webviewProxy.SOCKSURL(), nil
+	}
+	return d.webviewProxy.URL(), nil
+}
+
+// A revoked profile is retained until a fresh installation. Existing runtimes
+// retain this object, so reinstalling cannot let an old runtime recreate data.
+type extensionWebViewProfile struct {
+	mu       sync.Mutex
+	deleteMu sync.Mutex
+	revoked  bool
+	pages    map[*extensionWebViewPage]struct{}
+}
+
+type extensionWebViewPage struct {
+	enginewebview.Page
+	profile *extensionWebViewProfile
+	mu      sync.Mutex
+	closed  bool
+}
+
+func (p *extensionWebViewPage) Close() error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.closed {
+		return nil
+	}
+	if err := p.Page.Close(); err != nil {
+		return err
+	}
+	p.closed = true
+	p.profile.mu.Lock()
+	delete(p.profile.pages, p)
+	p.profile.mu.Unlock()
+	return nil
+}
+
+func (d *Downloader) extensionWebViewProfile(identity string) *extensionWebViewProfile {
+	d.webviewProfilesLock.Lock()
+	defer d.webviewProfilesLock.Unlock()
+	if d.webviewProfiles == nil {
+		d.webviewProfiles = make(map[string]*extensionWebViewProfile)
+	}
+	profile := d.webviewProfiles[identity]
+	if profile == nil {
+		profile = &extensionWebViewProfile{pages: make(map[*extensionWebViewPage]struct{})}
+		d.webviewProfiles[identity] = profile
+	}
+	return profile
+}
+
+func (d *Downloader) extensionWebViewLocation(identity string) (string, string, error) {
+	if identity == "" {
+		return "", "", fmt.Errorf("webview extension identity is required")
+	}
+	sum := sha256.Sum256([]byte(identity))
+	id := fmt.Sprintf("%x-%x-%x-%x-%x", sum[:4], sum[4:6], sum[6:8], sum[8:10], sum[10:16])
+	path, err := filepath.Abs(filepath.Join(d.cfg.StorageDir, "webview", fmt.Sprintf("%x", sum[:])))
+	return id, path, err
+}
+
+func (d *Downloader) removeExtensionWebViewProfile(identity string) error {
+	profile := d.extensionWebViewProfile(identity)
+	profile.deleteMu.Lock()
+	defer profile.deleteMu.Unlock()
+	profile.mu.Lock()
+	profile.revoked = true
+	pages := make([]*extensionWebViewPage, 0, len(profile.pages))
+	for page := range profile.pages {
+		pages = append(pages, page)
+	}
+	profile.mu.Unlock()
+	for _, page := range pages {
+		if err := page.Close(); err != nil {
+			return err
+		}
+	}
+	id, path, err := d.extensionWebViewLocation(identity)
+	if err != nil {
+		return err
+	}
+	if provider := d.cfg.WebViewProvider; provider != nil {
+		remover, ok := provider.(enginewebview.ProfileRemover)
+		if !ok {
+			// An unavailable backend cannot have created a profile this run.
+			// Still fail if a persisted native profile directory exists.
+			_, statErr := os.Stat(path)
+			if provider.IsAvailable() || len(pages) != 0 || !os.IsNotExist(statErr) {
+				return fmt.Errorf("WebView provider does not support profile removal")
+			}
+		} else if err := remover.RemoveProfile(id, path); err != nil {
+			return err
+		}
+	}
+	return os.RemoveAll(path)
+}

pkg/download/extension_webview_config_test.go

@@ -0,0 +1,51 @@
+package download
+
+import (
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"github.com/GopeedLab/gopeed/pkg/base"
+	enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
+)
+
+type profileCaptureProvider struct{ options []enginewebview.OpenOptions }
+
+func (*profileCaptureProvider) IsAvailable() bool { return true }
+func (p *profileCaptureProvider) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) {
+	p.options = append(p.options, opts)
+	return fakeRuntimeWebViewPage{}, nil
+}
+
+func TestWebViewProfileBoundToExtension(t *testing.T) {
+	root := t.TempDir()
+	d := &Downloader{cfg: &DownloaderConfig{StorageDir: root, DownloaderStoreConfig: &base.DownloaderStoreConfig{}}, configLock: &sync.RWMutex{}}
+	t.Cleanup(func() {
+		if d.webviewProxy != nil {
+			d.webviewProxy.Close()
+		}
+	})
+	provider := &profileCaptureProvider{}
+	for _, id := range []string{"extension-a", "extension-b", "extension-a", "../../escape"} {
+		opener := &extensionWebViewOpener{downloader: d, provider: provider, identity: id}
+		_, err := opener.Open(enginewebview.OpenOptions{ProfileID: "forged", DataPath: "/tmp/forged", ProxyURL: "http://wrong:80"})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	a, b, reopen := provider.options[0], provider.options[1], provider.options[2]
+	if a.ProfileID == b.ProfileID || a.DataPath == b.DataPath {
+		t.Fatal("different extensions shared browser storage")
+	}
+	if a.ProfileID != reopen.ProfileID || a.DataPath != reopen.DataPath {
+		t.Fatal("same extension lost profile identity")
+	}
+	for _, opts := range provider.options {
+		if filepath.Dir(opts.DataPath) != filepath.Join(root, "webview") {
+			t.Fatalf("profile escaped storage root: %s", opts.DataPath)
+		}
+		if opts.ProxyURL != d.webviewProxy.URL() && opts.ProxyURL != d.webviewProxy.SOCKSURL() {
+			t.Fatal("host proxy was overridden")
+		}
+	}
+}

pkg/download/extension_webview_remove_test.go

@@ -0,0 +1,189 @@
+package download
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"github.com/GopeedLab/gopeed/pkg/base"
+	webview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
+)
+
+type removalProvider struct {
+	pages   map[string][]*removalPage
+	removed []string
+	failure error
+	entered chan struct{}
+	resume  chan struct{}
+}
+type removalPage struct {
+	fakeRuntimeWebViewPage
+	closed bool
+}
+
+func (p *removalPage) Close() error          { p.closed = true; return nil }
+func (p *removalProvider) IsAvailable() bool { return true }
+func (p *removalProvider) Open(opts webview.OpenOptions) (webview.Page, error) {
+	if p.entered != nil {
+		close(p.entered)
+		<-p.resume
+	}
+	if p.pages == nil {
+		p.pages = make(map[string][]*removalPage)
+	}
+	page := &removalPage{}
+	p.pages[opts.ProfileID] = append(p.pages[opts.ProfileID], page)
+	return page, nil
+}
+func (p *removalProvider) RemoveProfile(id, path string) error {
+	for _, page := range p.pages[id] {
+		if !page.closed {
+			return errors.New("profile removed before closing pages")
+		}
+	}
+	if p.failure != nil {
+		return p.failure
+	}
+	p.removed = append(p.removed, id)
+	return nil
+}
+func removalDownloader(t *testing.T, provider *removalProvider) *Downloader {
+	t.Helper()
+	storage := NewMemStorage()
+	if err := storage.Setup([]string{bucketExtension, bucketExtensionStorage}); err != nil {
+		t.Fatal(err)
+	}
+	d := &Downloader{cfg: &DownloaderConfig{StorageDir: t.TempDir(), DownloaderStoreConfig: &base.DownloaderStoreConfig{}, WebViewProvider: provider}, storage: storage, configLock: &sync.RWMutex{}}
+	t.Cleanup(func() {
+		if d.webviewProxy != nil {
+			d.webviewProxy.Close()
+		}
+	})
+	return d
+}
+func profileSentinel(t *testing.T, d *Downloader, id string) string {
+	t.Helper()
+	_, path, err := d.extensionWebViewLocation(id)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(path, 0700); err != nil {
+		t.Fatal(err)
+	}
+	sentinel := filepath.Join(path, "cookies.sqlite")
+	if err := os.WriteFile(sentinel, []byte("private browser state"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	return sentinel
+}
+func TestDeleteExtensionRemovesWebViewData(t *testing.T) {
+	provider := &removalProvider{}
+	d := removalDownloader(t, provider)
+	ext, err := d.InstallExtensionByFolder("testdata/extensions/basic", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	identity := ext.Identity
+	opener := &extensionWebViewOpener{downloader: d, provider: provider, identity: identity, profile: d.extensionWebViewProfile(identity)}
+	for i := 0; i < 2; i++ {
+		if _, err := opener.Open(webview.OpenOptions{}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	sentinel := profileSentinel(t, d, identity)
+	other := profileSentinel(t, d, "other-extension")
+	// Updating the installation keeps both its profile object and stored data.
+	if _, err := d.InstallExtensionByFolder("testdata/extensions/basic", true); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(sentinel); err != nil {
+		t.Fatal("upgrade removed browser data", err)
+	}
+	if err := d.DeleteExtension(identity); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(sentinel); !os.IsNotExist(err) {
+		t.Fatalf("browser data retained: %v", err)
+	}
+	if _, err := os.Stat(other); err != nil {
+		t.Fatal("other extension data was removed", err)
+	}
+	if _, err := os.Stat(ext.DevPath); err != nil {
+		t.Fatal("dev source was removed", err)
+	}
+	if len(provider.removed) != 1 {
+		t.Fatal("native profile was not removed")
+	}
+	if _, err := opener.Open(webview.OpenOptions{}); err == nil {
+		t.Fatal("old runtime recreated removed profile")
+	}
+	if _, err := d.InstallExtensionByFolder("testdata/extensions/basic", true); err != nil {
+		t.Fatal(err)
+	}
+	fresh := &extensionWebViewOpener{downloader: d, provider: provider, identity: identity, profile: d.extensionWebViewProfile(identity)}
+	if _, err := fresh.Open(webview.OpenOptions{}); err != nil {
+		t.Fatal("reinstall could not open profile", err)
+	}
+	if _, err := opener.Open(webview.OpenOptions{}); err == nil {
+		t.Fatal("reinstall reactivated old runtime")
+	}
+}
+func TestDeleteExtensionProfileFailureIsRetryable(t *testing.T) {
+	provider := &removalProvider{failure: errors.New("native store busy")}
+	d := removalDownloader(t, provider)
+	ext, err := d.InstallExtensionByFolder("testdata/extensions/basic", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sentinel := profileSentinel(t, d, ext.Identity)
+	if err := d.DeleteExtension(ext.Identity); err == nil {
+		t.Fatal("cleanup failure was swallowed")
+	}
+	if _, err := d.GetExtension(ext.Identity); err != nil {
+		t.Fatal("extension removed before cleanup completed")
+	}
+	if _, err := os.Stat(sentinel); err != nil {
+		t.Fatal("profile path removed before native cleanup completed")
+	}
+	provider.failure = nil
+	if err := d.DeleteExtension(ext.Identity); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := d.GetExtension(ext.Identity); !errors.Is(err, ErrExtensionNotFound) {
+		t.Fatal("extension was not uninstalled")
+	}
+}
+func TestRemoveProfileWaitsForOpeningPage(t *testing.T) {
+	provider := &removalProvider{entered: make(chan struct{}), resume: make(chan struct{})}
+	d := removalDownloader(t, provider)
+	opener := &extensionWebViewOpener{downloader: d, provider: provider, identity: "race", profile: d.extensionWebViewProfile("race")}
+	opened := make(chan error, 1)
+	go func() { _, err := opener.Open(webview.OpenOptions{}); opened <- err }()
+	<-provider.entered
+	removed := make(chan error, 1)
+	go func() { removed <- d.removeExtensionWebViewProfile("race") }()
+	close(provider.resume)
+	if err := <-opened; err != nil {
+		t.Fatal(err)
+	}
+	if err := <-removed; err != nil {
+		t.Fatal(err)
+	}
+	if len(provider.removed) != 1 {
+		t.Fatal("profile not removed")
+	}
+}
+
+func TestUninstallWithoutAvailableWebView(t *testing.T) {
+	d := removalDownloader(t, &removalProvider{})
+	d.cfg.WebViewProvider = webview.NewUnavailableProvider()
+	ext, err := d.InstallExtensionByFolder("testdata/extensions/basic", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := d.DeleteExtension(ext.Identity); err != nil {
+		t.Fatal(err)
+	}
+}