fix: use server-side fetch header permissions for extensions

This commit is contained in:
Levi committed 2026-10-09 19:59:06 +08:00
1 parent fe2f45e40d
commit 342c55fbd6
6 files changed
+303 -89

No files matched your search

@@ -0,0 +1,222 @@
package engine_test
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/GopeedLab/gopeed/pkg/download/engine"
)
// Exercise server-side header permissions, including formerly forbidden names,
// download authentication, metadata and non-safelisted no-cors values.
var extensionRequestHeaders = [][2]string{
{"Accept-Charset", "utf-8"}, {"Accept-Encoding", "identity"},
{"Access-Control-Request-Headers", "authorization, x-download"},
{"Access-Control-Request-Method", "POST"}, {"Connection", "close"},
{"Connection", "keep-alive"},
{"Content-Length", "4"}, {"Cookie", "sid=manual"}, {"Cookie2", "legacy=1"},
{"Date", "Fri, 09 Oct 2026 00:00:00 GMT"}, {"DNT", "1"},
{"Host", "download.example:1234"}, {"Origin", "https://download.example"},
{"Permissions-Policy", "camera=()"}, {"Referer", "https://download.example/file"},
{"TE", "trailers"}, {"Trailer", "X-Checksum"}, {"Set-Cookie", "sid=request"},
{"Set-Cookie2", "legacy=request"}, {"Via", "1.1 proxy"},
{"Proxy-Authorization", "Basic dGVzdA=="}, {"Proxy-", "custom"},
{"Sec-Fetch-Site", "same-origin"}, {"Sec-Fetch-Mode", "custom"},
{"Sec-CH-UA", `"Download client"`}, {"Sec-", "custom"},
{"X-HTTP-Method", "TRACE"}, {"X-HTTP-Method-Override", "CONNECT"},
{"X-Method-Override", "TRACK"}, {"Authorization", "Bearer token"},
{"User-Agent", "Gopeed extension"}, {"Content-Type", "application/json"},
{"Range", "bytes=0-1,3-4"}, {"X-Download-Token", "custom"},
{"Accept-Language", strings.Repeat("x", 129)},
}
func TestExtensionRequestsSendNodeHeaders(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
body, err := io.ReadAll(request.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
name := request.URL.Query().Get("header")
value := request.Header.Get(name)
if name == "Host" {
value = request.Host
}
_ = json.NewEncoder(w).Encode([]string{value, string(body)})
}))
t.Cleanup(server.Close)
clients := []struct{ name, script string }{
{"fetch record", `const response = await fetch(url, {method: "POST", body: "test", headers: {[name]: value}}); return (await response.text()).trim();`},
{"fetch no-cors Headers", `const response = await fetch(url, {method: "POST", body: "test", mode: "no-cors", headers: new Headers([[name.toLowerCase(), value]])}); return (await response.text()).trim();`},
{"fetch Request clone", `const request = new Request(url, {method: "POST", body: "test", mode: "same-origin", headers: [[name.toUpperCase(), value]]}); const response = await fetch(request.clone()); return (await response.text()).trim();`},
}
for _, credentials := range []bool{false, true} {
clients = append(clients, struct{ name, script string }{
fmt.Sprintf("XHR credentials=%t", credentials),
fmt.Sprintf(`return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open("POST", url);
xhr.withCredentials = %t;
xhr.setRequestHeader(name.toLowerCase(), value);
xhr.onload = () => resolve(xhr.responseText.trim());
xhr.onerror = () => reject(new Error("XHR failed"));
xhr.send("test");
});`, credentials),
})
}
for _, header := range extensionRequestHeaders {
for _, client := range clients {
t.Run(header[0]+"/"+client.name, func(t *testing.T) {
runtime := engine.NewEngine(nil)
t.Cleanup(runtime.Close)
value, err := runtime.RunString(fmt.Sprintf(`(async () => {
const name = %q, value = %q;
const url = %q + "?header=" + encodeURIComponent(name);
%s
})()`, header[0], header[1], server.URL, client.script))
if err != nil {
t.Fatal(err)
}
want, _ := json.Marshal([]string{header[1], "test"})
if value != string(want) {
t.Fatalf("server received %#v, want %s", value, want)
}
})
}
}
}
func TestExtensionHeaderMutationsMatchNode(t *testing.T) {
headers := append([][2]string(nil), extensionRequestHeaders...)
// Node stores these in Headers/Request, but rejects them during fetch.
headers = append(headers, [2]string{"Expect", "100-continue"}, [2]string{"Keep-Alive", "timeout=5"}, [2]string{"Transfer-Encoding", "chunked"}, [2]string{"Upgrade", "websocket"})
data, _ := json.Marshal(headers)
runtime := engine.NewEngine(nil)
t.Cleanup(runtime.Close)
_, err := runtime.RunString(fmt.Sprintf(`(() => {
for (const mode of ["cors", "same-origin", "no-cors"]) {
for (const [name, value] of %s) {
const request = new Request("http://example.com", {mode, headers: {[name]: value}});
const headers = request.headers;
if (headers.get(name) !== value) throw new Error("Filtered init " + name + " in " + mode);
headers.delete(name.toUpperCase());
if (headers.has(name)) throw new Error("Cannot delete " + name);
headers.append(name, value);
if (headers.get(name) !== value) throw new Error("Cannot append " + name);
headers.set(name, value);
if (request.clone().headers.get(name) !== value) throw new Error("Cannot clone " + name);
}
}
return true;
})()`, data))
if err != nil {
t.Fatal(err)
}
}
func TestExtensionRequestsIgnoreUnsupportedTransportHeaders(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
body, err := io.ReadAll(request.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
value := request.Header.Get(request.URL.Query().Get("header"))
_ = json.NewEncoder(w).Encode([]string{value, string(body)})
}))
t.Cleanup(server.Close)
for _, header := range [][2]string{
{"Expect", "100-continue"}, {"Keep-Alive", "timeout=5"},
{"Transfer-Encoding", "chunked"}, {"Upgrade", "websocket"},
{"Connection", "upgrade"}, {"Content-Length", "3"},
{"Content-Length", "5"}, {"Content-Length", "invalid"},
{"Content-Length", "-1"}, {"Content-Length", ""},
} {
for _, xhr := range []bool{false, true} {
t.Run(fmt.Sprintf("%s=%s/XHR=%t", header[0], header[1], xhr), func(t *testing.T) {
runtime := engine.NewEngine(nil)
t.Cleanup(runtime.Close)
value, err := runtime.RunString(fmt.Sprintf(`(async () => {
const name = %q, value = %q;
const url = %q + "?header=" + encodeURIComponent(name);
if (%t) return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open("POST", url);
xhr.setRequestHeader(name, value);
xhr.onload = () => resolve(xhr.responseText.trim());
xhr.onerror = () => reject(new Error("XHR failed"));
xhr.send("test");
});
const response = await fetch(url, {method: "POST", body: "test", headers: {[name]: value}});
return (await response.text()).trim();
})()`, header[0], header[1], server.URL, xhr))
if err != nil {
t.Fatal(err)
}
wireValue := ""
if header[0] == "Content-Length" {
wireValue = "4"
}
want, _ := json.Marshal([]string{wireValue, "test"})
if value != string(want) {
t.Fatalf("server received %#v, want %s", value, want)
}
})
}
}
}
func TestExtensionResponseExposesSetCookie(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Add("Set-Cookie", "sid=one; HttpOnly")
w.Header().Add("Set-Cookie", "token=two")
w.Header().Set("Set-Cookie2", "legacy=three")
_, _ = w.Write([]byte("ok"))
}))
t.Cleanup(server.Close)
runtime := engine.NewEngine(nil)
t.Cleanup(runtime.Close)
value, err := runtime.RunString(fmt.Sprintf(`(async () => {
const response = await fetch(%q);
const clone = response.clone();
await response.text(); await clone.text();
let immutable = false;
try { response.headers.set("Set-Cookie", "changed=1"); } catch (error) { immutable = error instanceof TypeError; }
const synthetic = new Response(null, {headers: {"Set-Cookie": "initial=1"}});
synthetic.headers.append("Set-Cookie", "appended=2");
return JSON.stringify({cookies: clone.headers.getSetCookie(), legacy: response.headers.get("set-cookie2"), immutable, synthetic: synthetic.headers.getSetCookie()});
})()`, server.URL))
if err != nil {
t.Fatal(err)
}
const want = `{"cookies":["sid=one; HttpOnly","token=two"],"legacy":"legacy=three","immutable":true,"synthetic":["initial=1","appended=2"]}`
if value != want {
t.Fatalf("unexpected response headers: got %#v want %s", value, want)
}
}
func TestExtensionHeadersRejectInvalidSyntax(t *testing.T) {
runtime := engine.NewEngine(nil)
t.Cleanup(runtime.Close)
_, err := runtime.RunString(`(() => {
for (const [name, value] of [["Bad Header", "ok"], ["", "ok"], ["X:Header", "ok"],
["X-Test", "a\nb"], ["X-Test", "a\rb"], ["X-Test", "\0"], ["X-Test", "\u0100"]]) {
for (const create of [() => new Headers([[name, value]]),
() => new Request("http://example.com", {mode: "no-cors", headers: {[name]: value}}),
() => { const xhr = new XMLHttpRequest(); xhr.open("GET", "http://example.com"); xhr.setRequestHeader(name, value); }]) {
let rejected = false;
try { create(); } catch (error) { rejected = error instanceof TypeError; }
if (!rejected) throw new Error("Accepted invalid HTTP header syntax");
}
}
return true;
})()`)
if err != nil {
t.Fatal(err)
}
}
@@ -119,6 +119,7 @@ func TestExtensionRequestsSendExplicitCookie(t *testing.T) {
{"fetch record", `const response = await fetch(url, {headers: {Cookie: cookie}}); return response.text();`},
{"fetch Headers", `const response = await fetch(url, {headers: new Headers({cookie})}); return response.text();`},
{"fetch pairs", `const response = await fetch(url, {headers: [["cOoKiE", cookie]]}); return response.text();`},
{"fetch repeated Cookie", `const headers = new Headers([["Cookie", "sid=manual"], ["cookie", "token=provided"]]); if (headers.get("cookie") !== cookie) throw new Error("Incorrect Cookie separator"); const response = await fetch(url, {headers}); return response.text();`},
{"fetch Request clone", `const request = new Request(url, {headers: {Cookie: cookie}}); const response = await fetch(request.clone()); return response.text();`},
{"fetch Request set", `const request = new Request(url); request.headers.set("Cookie", cookie); const response = await fetch(request); return response.text();`},
{"fetch Request append", `const request = new Request(url); request.headers.append("Cookie", cookie); const response = await fetch(request); return response.text();`},
@@ -146,6 +147,15 @@ func TestExtensionRequestsSendExplicitCookie(t *testing.T) {
});`, withCredentials),
})
}
tests = append(tests, requestTest{"XHR repeated Cookie", `return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open("GET", url);
xhr.setRequestHeader("Cookie", "sid=manual");
xhr.setRequestHeader("cookie", "token=provided");
xhr.onload = () => resolve(xhr.responseText);
xhr.onerror = () => reject(new Error("XHR failed"));
xhr.send();
});`})
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
runtime := engine.NewEngine(nil)
+15 -63
View File
@@ -122,7 +122,7 @@
if (name === "set-cookie") {
for (const value of values) result.push([name, value]);
} else {
result.push([name, values.join(", ")]);
result.push([name, values.join(name === "cookie" ? "; " : ", ")]);
}
}
return result;
@@ -158,52 +158,10 @@
return iterator;
}
function forbiddenRequestHeader(name, value) {
if (name.startsWith("proxy-") || name.startsWith("sec-")) return true;
if ([
"accept-charset", "accept-encoding", "access-control-request-headers",
"access-control-request-method", "connection", "content-length", "cookie",
"cookie2", "date", "dnt", "expect", "host", "keep-alive", "origin",
"permissions-policy", "referer", "te", "trailer", "transfer-encoding",
"set-cookie", "upgrade", "via",
].includes(name)) return true;
if (["x-http-method", "x-http-method-override", "x-method-override"].includes(name)) {
return String(value).split(",").some((method) => {
method = method.trim().toUpperCase();
return method === "CONNECT" || method === "TRACE" || method === "TRACK";
});
}
return false;
}
function noCORSSafelistedRequestHeader(name, value) {
if (value.length > 128) return false;
const hasUnsafeByte = /[\x00-\x08\x0A-\x1F\x7F"():<>?@\[\\\]{}]/.test(value);
if (name === "accept") return !hasUnsafeByte;
if (name === "accept-language" || name === "content-language") {
return /^[0-9A-Za-z *,\-.;=]*$/.test(value);
}
if (name === "content-type") {
if (hasUnsafeByte) return false;
const essence = value.split(";", 1)[0].trim().toLowerCase();
return essence === "application/x-www-form-urlencoded" ||
essence === "multipart/form-data" || essence === "text/plain";
}
if (name === "range") return /^bytes=[0-9]+-[0-9]*$/.test(value);
return false;
}
function blockedByGuard(headers, name, value) {
function assertMutableHeaders(headers) {
// Match Node/Undici: HTTP syntax and immutability still apply, but browser
// forbidden-header and no-cors safelist rules do not apply to extensions.
if (headers._guard === "immutable") throw new TypeError("Headers are immutable");
// Extensions are privileged HTTP clients and may supply their own cookies,
// including in no-cors mode. Keep browser guards for other header names.
if (name === "cookie" && (headers._guard === "request" || headers._guard === "request-no-cors")) return false;
if (headers._guard === "request") return forbiddenRequestHeader(name, value);
if (headers._guard === "request-no-cors") {
return forbiddenRequestHeader(name, value) || !noCORSSafelistedRequestHeader(name, value);
}
if (headers._guard === "response") return name === "set-cookie" || name === "set-cookie2";
return false;
}
class Headers {
@@ -238,27 +196,21 @@
name = normalizeHeaderName(name);
value = normalizeHeaderValue(value);
const values = this._values.get(name);
const combinedValue = values && name !== "set-cookie" ? values.join(", ") + ", " + value : value;
if (blockedByGuard(this, name, combinedValue)) return;
assertMutableHeaders(this);
if (values) values.push(value);
else this._values.set(name, [value]);
}
delete(name) {
name = normalizeHeaderName(name);
if (this._guard === "immutable") throw new TypeError("Headers are immutable");
if (this._guard === "request-no-cors" && name !== "cookie") {
if (forbiddenRequestHeader(name, "") || !["accept", "accept-language", "content-language", "content-type", "range"].includes(name)) return;
this._values.delete(name);
return;
}
if (blockedByGuard(this, name, "")) return;
name = normalizeHeaderName(name);
assertMutableHeaders(this);
this._values.delete(name);
}
get(name) {
const values = this._values.get(normalizeHeaderName(name));
return values ? values.join(", ") : null;
name = normalizeHeaderName(name);
const values = this._values.get(name);
return values ? values.join(name === "cookie" ? "; " : ", ") : null;
}
getSetCookie() {
@@ -271,10 +223,10 @@
}
set(name, value) {
name = normalizeHeaderName(name);
value = normalizeHeaderValue(value);
if (blockedByGuard(this, name, value)) return;
this._values.set(name, [value]);
name = normalizeHeaderName(name);
value = normalizeHeaderValue(value);
assertMutableHeaders(this);
this._values.set(name, [value]);
}
keys() { return createHeadersIterator(this, "key"); }
@@ -487,7 +439,7 @@
}
this._duplex = "half";
const headerInit = init.headers !== undefined ? init.headers : (source ? source.headers : undefined);
this._headers = new Headers(headerInit, this._mode === "no-cors" ? "request-no-cors" : "request");
this._headers = new Headers(headerInit, "request");
let body = init.body !== undefined ? init.body : (source ? source._bodyInit : null);
if (source && !bodyWasOverridden && source.body) body = transferReadableStream(source.body);
const bodyIsStream = typeof ReadableStream !== "undefined" && body instanceof ReadableStream;
@@ -302,6 +302,23 @@ func (r *registry) Open(fingerprint string, proxyHandler func(*http.Request) (*u
return nil, err
}
for _, header := range metadata.Headers {
// Ignore transport controls unsupported by Node/Undici rather than
// applying a browser forbidden-header list or failing the whole request.
switch strings.ToLower(header[0]) {
case "expect", "keep-alive", "transfer-encoding", "upgrade", "content-length":
// Content-Length is generated from the buffered request body below.
continue
case "connection":
if value := strings.ToLower(header[1]); value != "close" && value != "keep-alive" {
continue
}
case "trailer":
// Fetch supplies a declaration, not a Go Request.Trailer payload.
// Keep its lower-case wire name so Go's writer does not omit it as
// the canonical Trailer field it normally generates itself.
httpRequest.Header["trailer"] = append(httpRequest.Header["trailer"], header[1])
continue
}
httpRequest.Header.Add(header[0], header[1])
}
if host := httpRequest.Header.Get("Host"); host != "" {
+24 -7
View File
@@ -9,7 +9,7 @@ The files under `fetch/` are copied from the official
Gopeed executes applicable worker-style Fetch API tests at the extension
JavaScript boundary. The checked-in runner currently executes unchanged WPT
tests for Headers, no-CORS header guards, Request construction and body
tests for Headers syntax and iteration, Request construction and body
consumption, ReadableStream bodies, keepalive, Request errors, Response
construction and body consumption, immutable network response headers, and
the Response static constructors.
@@ -23,12 +23,29 @@ keeps extension-specific behavior: `redirect: "manual"` exposes the HTTP
redirect response so download extensions can inspect `Location`, instead of
returning a browser `opaqueredirect` response.
Extensions may also supply an explicit `Cookie` request header, including in
`no-cors` mode. The runner excludes the single upstream assertion that treats
`Cookie` as an invalid request header; Gopeed integration tests instead verify
that fetch and XMLHttpRequest send it to the server. `credentials: "omit"`
disables automatic CookieJar cookies but preserves an explicit header. Other
request header guards remain covered by the unchanged upstream tests.
Header permissions follow [Node/Undici's server-side Fetch API](https://github.com/nodejs/undici/blob/main/README.md#forbidden-and-safelisted-header-names). Gopeed does not
apply browser forbidden request/response header lists or no-CORS safelists.
Extensions may supply authentication, referrer, origin, host, proxy and fetch
metadata headers, and read `Set-Cookie` response headers. Network response
headers remain immutable; invalid header syntax still raises an error.
Repeated `Cookie` values are combined with `; `, as in Node/Undici.
The runner excludes the browser-only `headers-no-cors.any.js` and
`headers-forbidden-override.any.js` suites, the filtering assertions within
`request-headers.any.js`, and the forbidden-response-cookie assertion within
`header-setcookie.any.js`. Gopeed's `extension_headers_test.go` replaces those
restrictions with integration tests that check headers actually received by
an HTTP server through fetch and XMLHttpRequest, plus header mutations,
cloning and response cookies. The upstream test files remain unchanged.
Gopeed deliberately ignores transport controls that Node/Undici rejects:
`Expect`, `Keep-Alive`, `Transfer-Encoding`, `Upgrade`, and `Connection` values
other than `close` or `keep-alive`. `Content-Length` is generated from the
actual request body, ignoring a supplied length. Explicit `Host` and
`Sec-Fetch-Mode` values are preserved rather than replaced by defaults.
These transport behaviors are also covered by integration tests.
`credentials: "omit"` disables automatic CookieJar cookies but preserves
explicit headers; Gopeed retains its extension CookieJar support.
`wpt_harness.js` is a small Goja adapter maintained by Gopeed; it is not copied
from WPT. Test logic below `fetch/` is unchanged; the checked-in copy may only
+15 -19
View File
@@ -20,9 +20,6 @@ var wptFetchUtils string
//go:embed testdata/wpt/fetch/api/request/request-error.js
var wptRequestError string
//go:embed testdata/wpt/fetch/api/cors/resources/not-cors-safelisted.json
var wptNoCORSSafelistedHeaders []byte
//go:embed testdata/wpt/fetch/api/headers/*.any.js
var wptHeaders embed.FS
@@ -30,31 +27,19 @@ var wptHeaders embed.FS
var wptRequestResponse embed.FS
func TestWPTHeaders(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) {
response.Header().Set("Content-Type", "application/json")
_, _ = response.Write(wptNoCORSSafelistedHeaders)
}))
t.Cleanup(server.Close)
tests := []string{
"header-setcookie.any.js",
"headers-basic.any.js",
"headers-casing.any.js",
"headers-combine.any.js",
"headers-errors.any.js",
"headers-forbidden-override.any.js",
"headers-normalize.any.js",
"headers-no-cors.any.js",
"headers-record.any.js",
"headers-structure.any.js",
}
for _, name := range tests {
t.Run(name, func(t *testing.T) {
setup := ""
if name == "headers-no-cors.any.js" {
setup = fmt.Sprintf("globalThis.location = new URL(%q);", server.URL+"/fetch/api/headers/test.any.js")
}
runWPTFile(t, wptHeaders, path.Join("testdata/wpt/fetch/api/headers", name), setup)
runWPTFile(t, wptHeaders, path.Join("testdata/wpt/fetch/api/headers", name), "")
})
}
}
@@ -112,12 +97,23 @@ func runWPTFile(t *testing.T, files embed.FS, name, setup string) {
t.Cleanup(runtime.Close)
adaptation := ""
if name == "testdata/wpt/fetch/api/request/request-headers.any.js" {
// Gopeed extensions intentionally allow explicit Cookie request headers.
// Keep the upstream file unchanged and exclude only its browser restriction.
// Node-style header permissions are covered by extension integration tests.
// Keep upstream tests unchanged and exclude browser-only restrictions.
adaptation = `
const upstreamTest = globalThis.test;
globalThis.test = function (callback, name) {
if (name === 'Adding invalid request header "Cookie: KO"') return;
if (name.startsWith('Adding invalid request header ') ||
name.startsWith('Adding invalid no-cors request header ') ||
name.startsWith('Check that request constructor is filtering headers ') ||
name.startsWith('Check that no-cors request constructor is filtering headers ')) return;
upstreamTest(callback, name);
};
`
} else if name == "testdata/wpt/fetch/api/headers/header-setcookie.any.js" {
adaptation = `
const upstreamTest = globalThis.test;
globalThis.test = function (callback, name) {
if (name === 'Set-Cookie is a forbidden response header') return;
upstreamTest(callback, name);
};
`