working on data verification

.gitignore

@@ -17,3 +17,6 @@ data/
 # OS generated files
 .DS_Store
 Thumbs.db
+
+# Environment Variables
+.env

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.autopep8"
+    },
+    "python.formatting.provider": "none"
+}

nba_data/scraper.py

@@ -5,6 +5,7 @@ import requests
 from bs4 import BeautifulSoup
 import csv
 import json
+import time
 
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
@@ -14,30 +15,36 @@ django.setup()
 
 from nba_data.models import PlayerData
 
-team = input('What team? ')
+teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHO']
+team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS']
+
+# team = input('What team? ') 
+#  OMIT above. Will now only ask for season and iterate thru teams list.
 season = input('What season? ')
 
-url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html'
+for team in teams:
+    url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html'
+    response = requests.get(url_make)
 
-url = 'https://www.basketball-reference.com/teams/HOU/2018.html'
-lakers = 'https://www.basketball-reference.com/teams/LAL/2020.html'
+    soup = BeautifulSoup(response.text, "html.parser")
     
-response = requests.get(url_make)
+    table = soup.find("table", {"id": "per_game"})
 
-soup = BeautifulSoup(response.text, "html.parser")
+    if table is None:
+        print(f"Table not found for team {team} and season {season}.")
+        sys.exit(1)
 
-table = soup.find("table", {"id": "per_game"})
-# rows = table.find_all("tr")
-header_row = table.find("thead").find("tr")
-rows = table.find("tbody").find_all("tr")
+    header_row = table.find("thead").find("tr")
+    rows = table.find("tbody").find_all("tr")
 
-# Extract coloumn headers
-headers = [col.text.strip() for col in header_row.find_all("th")][1:]
-headers.extend(['team', 'season'])  # Add 'team' and 'season' fields to headers
 
-# Extract rows as dictionaries
-data = []
-for row in rows:
+    # Extract coloumn headers
+    headers = [col.text.strip() for col in header_row.find_all("th")][1:]
+    headers.extend(['team', 'season'])  # Add 'team' and 'season' fields to headers
+
+    # Extract rows as dictionaries
+    data = []
+    for row in rows:
         cols = row.find_all("td")
         cols = [col.text.strip() for col in cols]
         row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)}
@@ -46,24 +53,24 @@ for row in rows:
         data.append(row_dict)
 
 
-# locate current directory
-script_dir = os.path.dirname(os.path.abspath(__file__))
+    # locate current directory
+    script_dir = os.path.dirname(os.path.abspath(__file__))
 
-# CSV file
-csv_output_path = os.path.join(script_dir, '..', 'data', 'output.csv')
-with open(csv_output_path, "w", newline="") as csvfile:
+    # CSV file
+    csv_output_path = os.path.join(script_dir, '..', 'data', 'output.csv')
+    with open(csv_output_path, "w", newline="") as csvfile:
         writer = csv.DictWriter(csvfile, fieldnames =headers)
         writer.writeheader()
         for row in data:
             writer.writerow(row)
 
-# Save as JSON
-json_output_path = os.path.join(script_dir, '..', 'data', 'output.json')
-with open(json_output_path, 'w') as jsonfile:
+    # Save as JSON
+    json_output_path = os.path.join(script_dir, '..', 'data', 'output.json')
+    with open(json_output_path, 'w') as jsonfile:
         json.dump(data, jsonfile)
 
-# save data to database
-for row in data:
+    # save data to database
+    for row in data:
         player_data = PlayerData(
             name=row['Player'],
             age=row['Age'],

nba_data/scraper_single.py

@@ -0,0 +1,102 @@
+import os
+import sys
+import django
+import requests
+from bs4 import BeautifulSoup
+import csv
+import json
+import time
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings')
+django.setup()
+
+from nba_data.models import PlayerData
+
+
+team = input('What team? ') 
+season = input('What season? ')
+
+
+url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html'
+response = requests.get(url_make)
+
+soup = BeautifulSoup(response.text, "html.parser")
+
+table = soup.find("table", {"id": "per_game"})
+
+if table is None:
+    print(f"Table not found for team {team} and season {season}.")
+    sys.exit(1)
+
+header_row = table.find("thead").find("tr")
+rows = table.find("tbody").find_all("tr")
+
+
+# Extract coloumn headers
+headers = [col.text.strip() for col in header_row.find_all("th")][1:]
+headers.extend(['team', 'season'])  # Add 'team' and 'season' fields to headers
+
+# Extract rows as dictionaries
+data = []
+for row in rows:
+    cols = row.find_all("td")
+    cols = [col.text.strip() for col in cols]
+    row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)}
+    row_dict['team'] = team
+    row_dict['season'] = season
+    data.append(row_dict)
+
+
+# locate current directory
+script_dir = os.path.dirname(os.path.abspath(__file__))
+
+# CSV file
+csv_output_path = os.path.join(script_dir, '..', 'data', 'output.csv')
+with open(csv_output_path, "w", newline="") as csvfile:
+    writer = csv.DictWriter(csvfile, fieldnames =headers)
+    writer.writeheader()
+    for row in data:
+        writer.writerow(row)
+
+# Save as JSON
+json_output_path = os.path.join(script_dir, '..', 'data', 'output.json')
+with open(json_output_path, 'w') as jsonfile:
+    json.dump(data, jsonfile)
+
+# save data to database
+for row in data:
+    player_data = PlayerData(
+        name=row['Player'],
+        age=row['Age'],
+        games=row['G'],
+        games_started=row['GS'],
+        minutes_pg=row['MP'],
+        field_goals=row['FG'],
+        field_attempts=row['FGA'],
+        field_percent=row['FG%'],
+        three_fg=row['3P'],
+        three_attempts=row['3PA'],
+        three_percent=row['3P%'],
+        two_fg=row['2P'],
+        two_attempts=row['2PA'],
+        two_percent=row['2P%'],
+        effect_fg_percent=row['eFG%'],
+        ft=row['FT'],
+        fta=row['FTA'],
+        ft_percent=row['FT%'],
+        ORB=row['ORB'],
+        DRB=row['DRB'],
+        TRB=row['TRB'],
+        AST=row['AST'],
+        STL=row['STL'],
+        BLK=row['BLK'],
+        TOV=row['TOV'],
+        PF=row['PF'],
+        PTS=row['PTS'],
+        team=row['team'],
+        season=row['season'],
+    )
+    player_data.save()

nba_stats/settings.py

@@ -11,6 +11,14 @@ https://docs.djangoproject.com/en/4.2/ref/settings/
 """
 
 from pathlib import Path
+from dotenv import load_dotenv
+load_dotenv()
+import os
+
+db_user = os.getenv('DB_USER')
+db_pass = os.getenv('DB_PASS')
+
+
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -76,8 +84,12 @@ WSGI_APPLICATION = 'nba_stats.wsgi.application'
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': BASE_DIR / 'db.sqlite3',
+        'ENGINE': 'django.db.backends.postgresql',
+        'NAME': 'nba_stats',
+        'USER': db_user,
+        'PASSWORD': db_pass,
+        'HOST': 'localhost',
+        'PORT': '',
     }
 }