fix: remove invalid characters in HTTP headers (#1053)

internal/protocol/http/fetcher.go

@@ -540,7 +540,7 @@ func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq
 		}
 		if len(extra.Header) > 0 {
 			for k, v := range extra.Header {
-				headers.Set(k, v)
+				headers.Set(k, strings.TrimSpace(v))
 			}
 		}
 		if extra.Body != "" {
@@ -548,7 +548,7 @@ func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq
 		}
 	}
 	if _, ok := headers[base.HttpHeaderUserAgent]; !ok {
-		headers.Set(base.HttpHeaderUserAgent, f.config.UserAgent)
+		headers.Set(base.HttpHeaderUserAgent, strings.TrimSpace(f.config.UserAgent))
 	}
 
 	if ctx != nil {

internal/protocol/http/fetcher_test.go

@@ -105,6 +105,21 @@ func TestFetcher_ResolveWithHostHeader(t *testing.T) {
 	}
 }
 
+func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
+	fetcher := buildFetcher()
+	err := fetcher.Resolve(&base.Request{
+		URL: "https://bing.com",
+		Extra: &http.ReqExtra{
+			Header: map[string]string{
+				"Referer": "\rtest",
+			},
+		},
+	})
+	if err != nil {
+		t.Errorf("Resolve() got = %v, want nil", err)
+	}
+}
+
 func testResolve(startTestServer func() net.Listener, path string, want *base.Resource, t *testing.T) {
 	listener := startTestServer()
 	defer listener.Close()