Compare commits
5
Commits
dev
..
0cf9e7d8ce
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0cf9e7d8ce | ||
|
|
717384ebfc | ||
|
|
22346b5748 | ||
|
|
4356b9b672 | ||
|
|
ba5d747371 |
No files matched your search
@@ -47,12 +47,6 @@ jobs:
|
|||||||
if: env.COOLIFY_WEBHOOK_URL != ''
|
if: env.COOLIFY_WEBHOOK_URL != ''
|
||||||
env:
|
env:
|
||||||
COOLIFY_WEBHOOK_URL: ${{ secrets.COOLIFY_WEBHOOK_URL }}
|
COOLIFY_WEBHOOK_URL: ${{ secrets.COOLIFY_WEBHOOK_URL }}
|
||||||
COOLIFY_API_TOKEN: ${{ secrets.COOLIFY_API_TOKEN }}
|
|
||||||
run: |
|
run: |
|
||||||
echo "==> Triggering Coolify deployment webhook..."
|
echo "==> Triggering Coolify deployment webhook..."
|
||||||
if [ -n "${COOLIFY_API_TOKEN:-}" ]; then
|
curl -fsSL -X POST "$COOLIFY_WEBHOOK_URL" || echo "Webhook trigger skipped or failed"
|
||||||
curl -fsSL -X POST -H "Authorization: Bearer $COOLIFY_API_TOKEN" "$COOLIFY_WEBHOOK_URL" || echo "Webhook trigger failed"
|
|
||||||
else
|
|
||||||
curl -fsSL -X POST "$COOLIFY_WEBHOOK_URL" || echo "Webhook trigger failed"
|
|
||||||
fi
|
|
||||||
|
|
||||||
@@ -1,75 +1,10 @@
|
|||||||
name: CI Health Check & Quality Gate
|
# .gitea/workflows/test.yml
|
||||||
|
name: test
|
||||||
on:
|
on: [push]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
- dev
|
|
||||||
pull_request:
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
healthcheck:
|
test:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
name: Code, dbt & Live App Health Check
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout Code
|
- uses: actions/checkout@v4
|
||||||
uses: actions/checkout@v4
|
- run: node --version
|
||||||
|
|
||||||
- name: Install System Dependencies
|
|
||||||
run: |
|
|
||||||
apt-get update && apt-get install -y curl unzip git ca-certificates
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: |
|
|
||||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
||||||
echo "$HOME/.local/bin" >> $GITHUB_PATH
|
|
||||||
echo "/root/.local/bin" >> $GITHUB_PATH
|
|
||||||
|
|
||||||
- name: Install Project Dependencies
|
|
||||||
run: |
|
|
||||||
export PATH="$HOME/.local/bin:/root/.local/bin:$PATH"
|
|
||||||
uv sync
|
|
||||||
|
|
||||||
- name: Python Syntax & Compilation Verification
|
|
||||||
run: |
|
|
||||||
export PATH="$HOME/.local/bin:/root/.local/bin:$PATH"
|
|
||||||
echo "==> Validating Python scripts compilation..."
|
|
||||||
uv run python -m py_compile streamlit_app/app.py scripts/db_storage.py
|
|
||||||
echo "✓ Python scripts compiled without errors"
|
|
||||||
|
|
||||||
- name: dbt Manifest & Model Validation
|
|
||||||
run: |
|
|
||||||
export PATH="$HOME/.local/bin:/root/.local/bin:$PATH"
|
|
||||||
echo "==> Validating dbt package dependencies..."
|
|
||||||
uv run dbt deps --project-dir dbt_nba --profiles-dir dbt_nba
|
|
||||||
echo "==> Validating dbt models, seeds, and tests compilation..."
|
|
||||||
uv run dbt parse --project-dir dbt_nba --profiles-dir dbt_nba
|
|
||||||
echo "✓ All dbt models, seeds, and macros validated"
|
|
||||||
|
|
||||||
- name: Cloudflare R2 Storage Probe (Optional if secrets present)
|
|
||||||
env:
|
|
||||||
S3_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }}
|
|
||||||
S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
|
|
||||||
S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
|
||||||
S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
|
||||||
S3_REGION: ${{ secrets.S3_REGION || 'auto' }}
|
|
||||||
S3_DB_KEY: ${{ secrets.S3_DB_KEY || 'dbt_nba.duckdb' }}
|
|
||||||
run: |
|
|
||||||
export PATH="$HOME/.local/bin:/root/.local/bin:$PATH"
|
|
||||||
if [ -n "${S3_ENDPOINT_URL:-}" ]; then
|
|
||||||
echo "==> Probing Cloudflare R2 storage status..."
|
|
||||||
uv run python scripts/db_storage.py status
|
|
||||||
else
|
|
||||||
echo "==> S3 secrets not configured for this step; skipping storage probe."
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Live Production App Health Probe
|
|
||||||
run: |
|
|
||||||
echo "==> Probing production endpoint https://streamlit.turbo-data.com/_stcore/health..."
|
|
||||||
HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" https://streamlit.turbo-data.com/_stcore/health || echo "000")
|
|
||||||
if [ "$HTTP_STATUS" = "200" ]; then
|
|
||||||
echo "✓ Production Streamlit dashboard is healthy (HTTP $HTTP_STATUS)"
|
|
||||||
else
|
|
||||||
echo "⚠️ Production health check returned HTTP $HTTP_STATUS (may still be starting or initializing)"
|
|
||||||
fi
|
|
||||||
@@ -1,132 +1,3 @@
|
|||||||
# 🏀 NBA Analytics — Streamlit & dbt Data Warehouse
|
# Streamlit NBA
|
||||||
|
|
||||||
[](https://git.turbo-data.com/nprasad2077/Streamlit_NBA/actions?workflow=pipeline.yaml)
|
NBA Streamlit Analytics
|
||||||
[](https://streamlit.turbo-data.com)
|
|
||||||
|
|
||||||
A modern NBA analytics data platform powered by **dbt**, **DuckDB**, and **Streamlit**. Transforms raw basketball event data into a dimensional star schema (32,000+ games, 480,000+ shots from 2002 to 2026), persisting artifacts to **Cloudflare R2 Object Storage** with automated **Gitea Actions** CI/CD and self-hosted **Coolify** deployment.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🏛️ Architecture
|
|
||||||
|
|
||||||
```mermaid
|
|
||||||
flowchart LR
|
|
||||||
PG[(Postgres Source)] -->|scripts/extract.sql| LOCAL_DB[(DuckDB 223 MB)]
|
|
||||||
LOCAL_DB -->|dbt build| MARTS[Marts Star Schema]
|
|
||||||
MARTS -->|db_storage.py upload| R2[(Cloudflare R2 Storage)]
|
|
||||||
R2 -->|Smart Sync| COOLIFY[Coolify: streamlit.turbo-data.com]
|
|
||||||
R2 -->|run_app.sh| LOCAL_APP[Local Streamlit Dashboard]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Medallion Data Modeling
|
|
||||||
|
|
||||||
| Layer | Models | Description |
|
|
||||||
|---|---|---|
|
|
||||||
| **Staging** | `stg_games`, `stg_line_scores`, `stg_player_game_basic_stats`, `stg_player_game_adv_stats`, `stg_player_shot_charts`, `stg_team_game_basic_stats`, `stg_team_game_adv_stats`, `stg_season_thresholds`, `stg_team_season_thresholds` | Clean raw source data, compute seasonal percentile thresholds, type casting |
|
|
||||||
| **Intermediate** | `int_games_enriched`, `int_player_performance`, `int_team_performance`, `int_player_shots_enriched` | Entity resolution across team history maps, shot chart & free throw unification |
|
|
||||||
| **Dimensions** | `dim_teams`, `dim_players`, `dim_seasons`, `dim_dates`, `dim_arenas`, `dim_shot_zones`, `dim_player_game_archetypes` | Conformed entity dimensions |
|
|
||||||
| **Facts** | `fct_game_results`, `fct_team_game_stats`, `fct_player_game_stats`, `fct_quarter_scoring`, `fct_player_shots`, `fct_player_game_shooting` | High-performance analytical fact tables with surrogate keys |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🚀 Quickstart & Local Setup
|
|
||||||
|
|
||||||
### 1. Prerequisites
|
|
||||||
- [uv](https://docs.astral.sh/uv/) (Python package manager)
|
|
||||||
- [DuckDB CLI](https://duckdb.org/docs/installation/) (v1.3.0+)
|
|
||||||
|
|
||||||
### 2. Clone and Install Dependencies
|
|
||||||
```bash
|
|
||||||
git clone https://git.turbo-data.com/nprasad2077/Streamlit_NBA.git
|
|
||||||
cd Streamlit_NBA
|
|
||||||
|
|
||||||
# Install Python 3.12 and dependencies into .venv
|
|
||||||
uv sync
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Configure Environment Variables
|
|
||||||
Copy the template and fill in your credentials:
|
|
||||||
```bash
|
|
||||||
cp .env.example .env
|
|
||||||
```
|
|
||||||
|
|
||||||
```ini
|
|
||||||
# Postgres Source Database (for ETL extraction)
|
|
||||||
POSTGRES_URL=postgresql://user:password@host:5432/nba
|
|
||||||
|
|
||||||
# Cloudflare R2 / S3 Object Storage
|
|
||||||
S3_ENDPOINT_URL=https://<ACCOUNT_ID>.r2.cloudflarestorage.com
|
|
||||||
S3_BUCKET_NAME=dbt-duckdb
|
|
||||||
S3_ACCESS_KEY_ID=your_r2_access_key
|
|
||||||
S3_SECRET_ACCESS_KEY=your_r2_secret_key
|
|
||||||
S3_REGION=auto
|
|
||||||
S3_DB_KEY=dbt_nba.duckdb
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Launch the Streamlit Dashboard
|
|
||||||
```bash
|
|
||||||
./scripts/run_app.sh
|
|
||||||
```
|
|
||||||
*The launcher automatically verifies and downloads the latest DuckDB database from Cloudflare R2 if it is not already present locally.*
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🗄️ Database Storage Management
|
|
||||||
|
|
||||||
Manage your analytical DuckDB file via the custom storage utility:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Check remote R2 object vs local file status
|
|
||||||
uv run python scripts/db_storage.py status
|
|
||||||
|
|
||||||
# Download latest database (skips if ETag/size already matches)
|
|
||||||
uv run python scripts/db_storage.py download
|
|
||||||
|
|
||||||
# Force re-download from Cloudflare R2
|
|
||||||
uv run python scripts/db_storage.py download --force
|
|
||||||
|
|
||||||
# Upload local database to Cloudflare R2
|
|
||||||
uv run python scripts/db_storage.py upload
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ⚙️ Running the ETL Pipeline
|
|
||||||
|
|
||||||
To execute the complete ETL process locally (Extract from Postgres $\rightarrow$ Run dbt transformations $\rightarrow$ Upload artifact to Cloudflare R2):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/pipeline.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🛠️ dbt Development Commands
|
|
||||||
|
|
||||||
Run dbt models and tests directly using `uv`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Install dbt packages (dbt_utils)
|
|
||||||
uv run dbt deps
|
|
||||||
|
|
||||||
# Parse and validate project manifest
|
|
||||||
uv run dbt parse
|
|
||||||
|
|
||||||
# Build all models, seeds, and tests
|
|
||||||
uv run dbt build
|
|
||||||
|
|
||||||
# Build specific layer by tag
|
|
||||||
uv run dbt build --select "tag:staging"
|
|
||||||
uv run dbt build --select "tag:intermediate"
|
|
||||||
uv run dbt build --select "tag:marts"
|
|
||||||
|
|
||||||
# Run tests only
|
|
||||||
uv run dbt test
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🚢 CI/CD & Deployment
|
|
||||||
|
|
||||||
- **Gitea Actions (`.gitea/workflows/pipeline.yaml`)**: Runs scheduled daily ETL builds (06:00 UTC) on a custom `ubuntu-latest` runner and updates Cloudflare R2.
|
|
||||||
- **Coolify Self-Hosting ([streamlit.turbo-data.com](https://streamlit.turbo-data.com))**: Production deployment built via Dockerfile with automated Let's Encrypt SSL and persistent storage caching.
|
|
||||||
+4
-77
@@ -8,55 +8,6 @@ from pathlib import Path
|
|||||||
|
|
||||||
st.set_page_config(page_title="NBA Analytics", page_icon="🏀", layout="wide")
|
st.set_page_config(page_title="NBA Analytics", page_icon="🏀", layout="wide")
|
||||||
|
|
||||||
# Custom Responsive Sidebar & Navigation Styling
|
|
||||||
st.markdown("""
|
|
||||||
<style>
|
|
||||||
/* Sidebar Navigation Cards */
|
|
||||||
[data-testid="stSidebar"] [data-testid="stRadio"] > div {
|
|
||||||
gap: 0.35rem;
|
|
||||||
}
|
|
||||||
[data-testid="stSidebar"] [data-testid="stRadio"] label {
|
|
||||||
background-color: rgba(128, 128, 128, 0.06);
|
|
||||||
border: 1px solid rgba(128, 128, 128, 0.12);
|
|
||||||
border-radius: 8px;
|
|
||||||
padding: 0.6rem 0.85rem;
|
|
||||||
margin-bottom: 0.15rem;
|
|
||||||
cursor: pointer !important;
|
|
||||||
transition: all 0.15s ease-in-out;
|
|
||||||
width: 100%;
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
user-select: none;
|
|
||||||
}
|
|
||||||
[data-testid="stSidebar"] [data-testid="stRadio"] label:hover {
|
|
||||||
background-color: rgba(128, 128, 128, 0.14);
|
|
||||||
border-color: rgba(255, 107, 0, 0.4);
|
|
||||||
transform: translateX(3px);
|
|
||||||
}
|
|
||||||
[data-testid="stSidebar"] [data-testid="stRadio"] label:has(input:checked),
|
|
||||||
[data-testid="stSidebar"] [data-testid="stRadio"] label[data-checked="true"] {
|
|
||||||
background: linear-gradient(90deg, rgba(255, 107, 0, 0.22), rgba(255, 107, 0, 0.06)) !important;
|
|
||||||
border-color: #ff6b00 !important;
|
|
||||||
font-weight: 600;
|
|
||||||
box-shadow: 0 0 10px rgba(255, 107, 0, 0.15);
|
|
||||||
}
|
|
||||||
/* Hide the default radio circle for cleaner button appearance */
|
|
||||||
[data-testid="stSidebar"] [data-testid="stRadio"] label input[type="radio"] {
|
|
||||||
accent-color: #ff6b00;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Season Selectbox Hit Area & Styling */
|
|
||||||
[data-testid="stSidebar"] [data-testid="stSelectbox"] div[data-baseweb="select"] {
|
|
||||||
cursor: pointer !important;
|
|
||||||
border-radius: 8px;
|
|
||||||
min-height: 42px;
|
|
||||||
}
|
|
||||||
[data-testid="stSidebar"] [data-testid="stSelectbox"] div[data-baseweb="select"]:hover {
|
|
||||||
border-color: #ff6b00;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
""", unsafe_allow_html=True)
|
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
# CONNECTION & QUERY INFRASTRUCTURE
|
# CONNECTION & QUERY INFRASTRUCTURE
|
||||||
# ============================================================
|
# ============================================================
|
||||||
@@ -95,47 +46,23 @@ seasons = get_seasons()
|
|||||||
season_options = seasons["season_start_year"].tolist()
|
season_options = seasons["season_start_year"].tolist()
|
||||||
season_labels = dict(zip(seasons["season_start_year"], seasons["season_display"]))
|
season_labels = dict(zip(seasons["season_start_year"], seasons["season_display"]))
|
||||||
|
|
||||||
PAGE_OPTIONS = [
|
|
||||||
"Overview",
|
|
||||||
"Teams",
|
|
||||||
"Player Lab",
|
|
||||||
"Shot Charts",
|
|
||||||
"Head to Head",
|
|
||||||
"Game Trends",
|
|
||||||
"Quarter Analysis",
|
|
||||||
]
|
|
||||||
|
|
||||||
PAGE_ICONS = {
|
|
||||||
"Overview": "📊 Overview",
|
|
||||||
"Teams": "🛡️ Teams",
|
|
||||||
"Player Lab": "🧪 Player Lab",
|
|
||||||
"Shot Charts": "🎯 Shot Charts",
|
|
||||||
"Head to Head": "⚔️ Head to Head",
|
|
||||||
"Game Trends": "📈 Game Trends",
|
|
||||||
"Quarter Analysis": "⏱️ Quarter Analysis",
|
|
||||||
}
|
|
||||||
|
|
||||||
if "season" not in st.session_state:
|
if "season" not in st.session_state:
|
||||||
st.session_state["season"] = season_options[0]
|
st.session_state["season"] = season_options[0]
|
||||||
|
|
||||||
st.sidebar.title("🏀 NBA Analytics")
|
st.sidebar.title("🏀 NBA Analytics")
|
||||||
|
|
||||||
# Direct key binding eliminates dropped clicks and rerun desync
|
st.session_state["season"] = st.sidebar.selectbox(
|
||||||
st.sidebar.selectbox(
|
|
||||||
"Season",
|
"Season",
|
||||||
season_options,
|
season_options,
|
||||||
key="season",
|
index=season_options.index(st.session_state["season"]),
|
||||||
format_func=lambda x: season_labels.get(x, str(x)),
|
format_func=lambda x: season_labels[x],
|
||||||
)
|
)
|
||||||
|
|
||||||
st.sidebar.divider()
|
st.sidebar.divider()
|
||||||
|
|
||||||
# Full-width clickable navigation cards
|
|
||||||
page = st.sidebar.radio(
|
page = st.sidebar.radio(
|
||||||
"Navigate",
|
"Navigate",
|
||||||
PAGE_OPTIONS,
|
["Overview", "Teams", "Player Lab", "Shot Charts", "Head to Head", "Game Trends", "Quarter Analysis"],
|
||||||
format_func=lambda x: PAGE_ICONS.get(x, x),
|
|
||||||
key="current_page",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Convenience accessors
|
# Convenience accessors
|
||||||
|
|||||||
Reference in new issue
Block a user