43 lines
1.3 KiB
YAML
43 lines
1.3 KiB
YAML
name: NBA Data Pipeline
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 6 * * *'
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
build-and-sync:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup uv
|
|
uses: astral-sh/setup-uv@v4
|
|
with:
|
|
version: "latest"
|
|
|
|
- name: Install DuckDB CLI
|
|
run: |
|
|
curl -fsSL https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-linux-amd64.zip -o duckdb.zip
|
|
unzip -o duckdb.zip
|
|
sudo mv duckdb /usr/local/bin/
|
|
rm -f duckdb.zip
|
|
|
|
- name: Install gettext (envsubst)
|
|
run: |
|
|
sudo apt-get update && sudo apt-get install -y gettext-base
|
|
|
|
- name: Run ETL Pipeline & Upload to Cloudflare R2
|
|
env:
|
|
POSTGRES_URL: ${{ secrets.POSTGRES_URL }}
|
|
S3_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }}
|
|
S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
|
|
S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
|
S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
|
S3_REGION: ${{ secrets.S3_REGION || 'auto' }}
|
|
S3_DB_KEY: ${{ secrets.S3_DB_KEY || 'dbt_nba.duckdb' }}
|
|
run: |
|
|
chmod +x ./scripts/pipeline.sh ./scripts/db_storage.py
|
|
./scripts/pipeline.sh
|