complete app

scripts/pipeline.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+DB_PATH="$REPO_ROOT/data/DB/dbt_nba.duckdb"
+REPORTS_DB="$REPO_ROOT/dbt_nba/reports/sources/nba/dbt_nba.duckdb"
+
+# Load .env if it exists
+if [ -f "$REPO_ROOT/.env" ]; then
+  set -a
+  source "$REPO_ROOT/.env"
+  set +a
+fi
+
+# Check prerequisites
+if [ -z "${POSTGRES_URL:-}" ]; then
+  echo "ERROR: POSTGRES_URL environment variable is not set" >&2
+  echo "Please export POSTGRES_URL or configure it in $REPO_ROOT/.env" >&2
+  exit 1
+fi
+
+echo "==> [1/4] Extracting from Postgres into DuckDB..."
+mkdir -p "$(dirname "$DB_PATH")"
+mkdir -p "$(dirname "$REPORTS_DB")"
+rm -f "$DB_PATH"
+envsubst < "$REPO_ROOT/scripts/extract.sql" | duckdb "$DB_PATH"
+
+echo "==> [2/4] Running dbt build --full-refresh..."
+export DBT_PROJECT_DIR="$REPO_ROOT/dbt_nba"
+export DBT_PROFILES_DIR="$REPO_ROOT/dbt_nba"
+export DBT_DUCKDB_PATH="$DB_PATH"
+
+uv run --project "$REPO_ROOT" dbt deps --project-dir "$DBT_PROJECT_DIR" --profiles-dir "$DBT_PROFILES_DIR"
+uv run --project "$REPO_ROOT" dbt build --full-refresh --project-dir "$DBT_PROJECT_DIR" --profiles-dir "$DBT_PROFILES_DIR"
+
+echo "==> [3/4] Copying database to reports directory..."
+cp "$DB_PATH" "$REPORTS_DB"
+
+echo "==> [4/4] Syncing database artifact to Cloudflare R2 / S3..."
+if [ -n "${S3_ENDPOINT_URL:-}" ] || [ -n "${R2_ENDPOINT_URL:-}" ]; then
+  uv run --project "$REPO_ROOT" python "$REPO_ROOT/scripts/db_storage.py" upload
+  echo "==> Cloudflare R2 upload complete!"
+else
+  echo "==> S3_ENDPOINT_URL not set; skipping remote upload (local DB ready)."
+fi
+
+echo "==> Pipeline complete successfully!"