feat: show extraction progress (#1217)

internal/protocol/http/fetcher_test.go

@@ -114,45 +114,58 @@ func TestFetcher_Resolve(t *testing.T) {
 		},
 	}, t)
 
+	// Test URL without file path - should use domain/host as filename
+	listener := test.StartTestRootServer()
+	defer listener.Close()
 	fetcher := buildFetcher()
 	err := fetcher.Resolve(&base.Request{
-		URL: "http://github.com",
+		URL: "http://" + listener.Addr().String() + "/",
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	if fetcher.Meta().Res.Files[0].Name != "github.com" {
-		t.Errorf("Resolve() got = %v, want %v", fetcher.Meta().Res, "github.com")
+	// When no filename is provided, it should use the hostname (without port) as the name
+	expectedName := "127.0.0.1"
+	if fetcher.Meta().Res.Files[0].Name != expectedName {
+		t.Errorf("Resolve() got name = %v, want %v", fetcher.Meta().Res.Files[0].Name, expectedName)
 	}
 }
 
 func TestFetcher_ResolveWithHostHeader(t *testing.T) {
+	listener := test.StartTestHostHeaderServer()
+	defer listener.Close()
+	
 	fetcher := buildFetcher()
 	err := fetcher.Resolve(&base.Request{
-		URL: "https://bing.com",
+		URL: "http://" + listener.Addr().String() + "/",
 		Extra: &http.ReqExtra{
 			Header: map[string]string{
 				"Host": "test",
 			},
 		},
 	})
+	// The server should return 400 for invalid Host header
 	if err == nil || !strings.Contains(err.Error(), "400") {
-		t.Errorf("Resolve() got = %v, want %v", err, "400")
+		t.Errorf("Resolve() got = %v, want error containing 400", err)
 	}
 }
 
 func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
+	listener := test.StartTestRootServer()
+	defer listener.Close()
+	
 	fetcher := buildFetcher()
 	err := fetcher.Resolve(&base.Request{
-		URL: "https://bing.com",
+		URL: "http://" + listener.Addr().String() + "/",
 		Extra: &http.ReqExtra{
 			Header: map[string]string{
 				"Referer": "\rtest",
 			},
 		},
 	})
+	// Invalid header with \r should be sanitized by Go's http client, allowing the request to succeed
 	if err != nil {
-		t.Errorf("Resolve() got = %v, want nil", err)
+		t.Errorf("Resolve() got = %v, want nil (invalid headers should be sanitized)", err)
 	}
 }
 

internal/test/httptest.go

@@ -173,6 +173,39 @@ func StartTestCustomServer() net.Listener {
 	})
 }
 
+// StartTestHostHeaderServer starts a server that validates the Host header
+// Returns 400 Bad Request if the Host header value equals "test"
+func StartTestHostHeaderServer() net.Listener {
+	return startTestServer(func(sl *shutdownListener) http.Handler {
+		mux := http.NewServeMux()
+		mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
+			// If the Host header is "test", return 400 (simulating server that validates Host)
+			if request.Host == "test" {
+				writer.WriteHeader(400)
+				writer.Write([]byte("Bad Request: Invalid Host header"))
+				return
+			}
+			writer.WriteHeader(200)
+			writer.Write([]byte("OK"))
+		})
+		return mux
+	})
+}
+
+// StartTestRootServer starts a simple server at the root path
+// Used to test URL resolution when no filename is provided
+func StartTestRootServer() net.Listener {
+	return startTestServer(func(sl *shutdownListener) http.Handler {
+		mux := http.NewServeMux()
+		mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
+			writer.Header().Set("Content-Type", "text/html")
+			writer.WriteHeader(200)
+			writer.Write([]byte("<html><body>Test Page</body></html>"))
+		})
+		return mux
+	})
+}
+
 func StartTestRetryServer() net.Listener {
 	counter := 0
 	return startTestServer(func(sl *shutdownListener) http.Handler {