45 lines
1.4 KiB
YAML
45 lines
1.4 KiB
YAML
name: NBA Data Pipeline
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 6 * * *'
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
build-and-sync:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Install System Dependencies
|
|
run: |
|
|
apt-get update && apt-get install -y curl unzip gettext-base ca-certificates git
|
|
|
|
- name: Install uv
|
|
run: |
|
|
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
echo "$HOME/.local/bin" >> $GITHUB_PATH
|
|
echo "/root/.local/bin" >> $GITHUB_PATH
|
|
|
|
- name: Install DuckDB CLI
|
|
run: |
|
|
curl -fsSL https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-linux-amd64.zip -o duckdb.zip
|
|
unzip -o duckdb.zip
|
|
mv duckdb /usr/local/bin/
|
|
rm -f duckdb.zip
|
|
|
|
- name: Run ETL Pipeline & Upload to Cloudflare R2
|
|
env:
|
|
POSTGRES_URL: ${{ secrets.POSTGRES_URL }}
|
|
S3_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }}
|
|
S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
|
|
S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
|
S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
|
S3_REGION: ${{ secrets.S3_REGION || 'auto' }}
|
|
S3_DB_KEY: ${{ secrets.S3_DB_KEY || 'dbt_nba.duckdb' }}
|
|
run: |
|
|
export PATH="$HOME/.local/bin:/root/.local/bin:$PATH"
|
|
chmod +x ./scripts/pipeline.sh ./scripts/db_storage.py
|
|
./scripts/pipeline.sh
|