fix

scripts/extract_data.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+"""
+Extract source tables from Postgres into DuckDB using the DuckDB Python library.
+Reads SQL commands from scripts/extract.sql with environment variable interpolation.
+"""
+import os
+import sys
+from pathlib import Path
+import duckdb
+
+def main():
+    repo_root = Path(__file__).resolve().parent.parent
+    db_path = repo_root / "data" / "DB" / "dbt_nba.duckdb"
+    sql_path = repo_root / "scripts" / "extract.sql"
+    
+    postgres_url = os.getenv("POSTGRES_URL")
+    if not postgres_url:
+        print("ERROR: POSTGRES_URL environment variable is not set", file=sys.stderr)
+        print(f"Please export POSTGRES_URL or configure it in {repo_root}/.env", file=sys.stderr)
+        sys.exit(1)
+        
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    if db_path.exists():
+        db_path.unlink()
+        
+    with open(sql_path, "r", encoding="utf-8") as f:
+        sql = f.read()
+        
+    # Replace variable
+    sql = sql.replace("${POSTGRES_URL}", postgres_url)
+    
+    print(f"==> Connecting to DuckDB at {db_path}...")
+    con = duckdb.connect(str(db_path))
+    try:
+        print("==> Executing extraction queries...")
+        con.execute(sql)
+        print("==> Extraction complete successfully!")
+    finally:
+        con.close()
+
+if __name__ == "__main__":
+    main()

scripts/pipeline.sh

@@ -23,7 +23,7 @@ echo "==> [1/4] Extracting from Postgres into DuckDB..."
 mkdir -p "$(dirname "$DB_PATH")"
 mkdir -p "$(dirname "$REPORTS_DB")"
 rm -f "$DB_PATH"
-envsubst < "$REPO_ROOT/scripts/extract.sql" | duckdb "$DB_PATH"
+uv run --project "$REPO_ROOT" python "$REPO_ROOT/scripts/extract_data.py"
 
 echo "==> [2/4] Running dbt build --full-refresh..."
 export DBT_PROJECT_DIR="$REPO_ROOT/dbt_nba"