48 lines
1.6 KiB
Bash
Executable File
48 lines
1.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
DB_PATH="$REPO_ROOT/data/DB/dbt_nba.duckdb"
|
|
REPORTS_DB="$REPO_ROOT/dbt_nba/reports/sources/nba/dbt_nba.duckdb"
|
|
|
|
# Load .env if it exists
|
|
if [ -f "$REPO_ROOT/.env" ]; then
|
|
set -a
|
|
source "$REPO_ROOT/.env"
|
|
set +a
|
|
fi
|
|
|
|
# Check prerequisites
|
|
if [ -z "${POSTGRES_URL:-}" ]; then
|
|
echo "ERROR: POSTGRES_URL environment variable is not set" >&2
|
|
echo "Please export POSTGRES_URL or configure it in $REPO_ROOT/.env" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "==> [1/4] Extracting from Postgres into DuckDB..."
|
|
mkdir -p "$(dirname "$DB_PATH")"
|
|
mkdir -p "$(dirname "$REPORTS_DB")"
|
|
rm -f "$DB_PATH"
|
|
envsubst < "$REPO_ROOT/scripts/extract.sql" | duckdb "$DB_PATH"
|
|
|
|
echo "==> [2/4] Running dbt build --full-refresh..."
|
|
export DBT_PROJECT_DIR="$REPO_ROOT/dbt_nba"
|
|
export DBT_PROFILES_DIR="$REPO_ROOT/dbt_nba"
|
|
export DBT_DUCKDB_PATH="$DB_PATH"
|
|
|
|
uv run --project "$REPO_ROOT" dbt deps --project-dir "$DBT_PROJECT_DIR" --profiles-dir "$DBT_PROFILES_DIR"
|
|
uv run --project "$REPO_ROOT" dbt build --full-refresh --project-dir "$DBT_PROJECT_DIR" --profiles-dir "$DBT_PROFILES_DIR"
|
|
|
|
echo "==> [3/4] Copying database to reports directory..."
|
|
cp "$DB_PATH" "$REPORTS_DB"
|
|
|
|
echo "==> [4/4] Syncing database artifact to Cloudflare R2 / S3..."
|
|
if [ -n "${S3_ENDPOINT_URL:-}" ] || [ -n "${R2_ENDPOINT_URL:-}" ]; then
|
|
uv run --project "$REPO_ROOT" python "$REPO_ROOT/scripts/db_storage.py" upload
|
|
echo "==> Cloudflare R2 upload complete!"
|
|
else
|
|
echo "==> S3_ENDPOINT_URL not set; skipping remote upload (local DB ready)."
|
|
fi
|
|
|
|
echo "==> Pipeline complete successfully!"
|