Merge pull request #3 from nprasad2077/dev

working on data verification

.gitignore

@@ -17,3 +17,6 @@ data/
 # OS generated files
 .DS_Store
 Thumbs.db
+
+# Environment Variables
+.env

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.autopep8"
+    },
+    "python.formatting.provider": "none"
+}

nba_data/scraper.py

@@ -5,6 +5,7 @@ import requests
 from bs4 import BeautifulSoup
 import csv
 import json
+import time
 
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
@@ -14,85 +15,91 @@ django.setup()
 
 from nba_data.models import PlayerData
 
-team = input('What team? ')
+teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHO']
+team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS']
+
+# team = input('What team? ') 
+#  OMIT above. Will now only ask for season and iterate thru teams list.
 season = input('What season? ')
 
-url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html'
+for team in teams:
+    url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html'
+    response = requests.get(url_make)
 
-url = 'https://www.basketball-reference.com/teams/HOU/2018.html'
+    soup = BeautifulSoup(response.text, "html.parser")
-lakers = 'https://www.basketball-reference.com/teams/LAL/2020.html'
+    
+    table = soup.find("table", {"id": "per_game"})
 
-response = requests.get(url_make)
+    if table is None:
+        print(f"Table not found for team {team} and season {season}.")
+        sys.exit(1)
 
-soup = BeautifulSoup(response.text, "html.parser")
+    header_row = table.find("thead").find("tr")
-
+    rows = table.find("tbody").find_all("tr")
-table = soup.find("table", {"id": "per_game"})
-# rows = table.find_all("tr")
-header_row = table.find("thead").find("tr")
-rows = table.find("tbody").find_all("tr")
-
-# Extract coloumn headers
-headers = [col.text.strip() for col in header_row.find_all("th")][1:]
-headers.extend(['team', 'season'])  # Add 'team' and 'season' fields to headers
-
-# Extract rows as dictionaries
-data = []
-for row in rows:
-    cols = row.find_all("td")
-    cols = [col.text.strip() for col in cols]
-    row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)}
-    row_dict['team'] = team
-    row_dict['season'] = season
-    data.append(row_dict)
 
 
-# locate current directory
+    # Extract coloumn headers
-script_dir = os.path.dirname(os.path.abspath(__file__))
+    headers = [col.text.strip() for col in header_row.find_all("th")][1:]
+    headers.extend(['team', 'season'])  # Add 'team' and 'season' fields to headers
 
-# CSV file
+    # Extract rows as dictionaries
-csv_output_path = os.path.join(script_dir, '..', 'data', 'output.csv')
+    data = []
-with open(csv_output_path, "w", newline="") as csvfile:
+    for row in rows:
-    writer = csv.DictWriter(csvfile, fieldnames =headers)
+        cols = row.find_all("td")
-    writer.writeheader()
+        cols = [col.text.strip() for col in cols]
+        row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)}
+        row_dict['team'] = team
+        row_dict['season'] = season
+        data.append(row_dict)
+
+
+    # locate current directory
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+
+    # CSV file
+    csv_output_path = os.path.join(script_dir, '..', 'data', 'output.csv')
+    with open(csv_output_path, "w", newline="") as csvfile:
+        writer = csv.DictWriter(csvfile, fieldnames =headers)
+        writer.writeheader()
+        for row in data:
+            writer.writerow(row)
+
+    # Save as JSON
+    json_output_path = os.path.join(script_dir, '..', 'data', 'output.json')
+    with open(json_output_path, 'w') as jsonfile:
+        json.dump(data, jsonfile)
+
+    # save data to database
     for row in data:
-        writer.writerow(row)
+        player_data = PlayerData(
-
+            name=row['Player'],
-# Save as JSON
+            age=row['Age'],
-json_output_path = os.path.join(script_dir, '..', 'data', 'output.json')
+            games=row['G'],
-with open(json_output_path, 'w') as jsonfile:
+            games_started=row['GS'],
-    json.dump(data, jsonfile)
+            minutes_pg=row['MP'],
-
+            field_goals=row['FG'],
-# save data to database
+            field_attempts=row['FGA'],
-for row in data:
+            field_percent=row['FG%'],
-    player_data = PlayerData(
+            three_fg=row['3P'],
-        name=row['Player'],
+            three_attempts=row['3PA'],
-        age=row['Age'],
+            three_percent=row['3P%'],
-        games=row['G'],
+            two_fg=row['2P'],
-        games_started=row['GS'],
+            two_attempts=row['2PA'],
-        minutes_pg=row['MP'],
+            two_percent=row['2P%'],
-        field_goals=row['FG'],
+            effect_fg_percent=row['eFG%'],
-        field_attempts=row['FGA'],
+            ft=row['FT'],
-        field_percent=row['FG%'],
+            fta=row['FTA'],
-        three_fg=row['3P'],
+            ft_percent=row['FT%'],
-        three_attempts=row['3PA'],
+            ORB=row['ORB'],
-        three_percent=row['3P%'],
+            DRB=row['DRB'],
-        two_fg=row['2P'],
+            TRB=row['TRB'],
-        two_attempts=row['2PA'],
+            AST=row['AST'],
-        two_percent=row['2P%'],
+            STL=row['STL'],
-        effect_fg_percent=row['eFG%'],
+            BLK=row['BLK'],
-        ft=row['FT'],
+            TOV=row['TOV'],
-        fta=row['FTA'],
+            PF=row['PF'],
-        ft_percent=row['FT%'],
+            PTS=row['PTS'],
-        ORB=row['ORB'],
+            team=row['team'],
-        DRB=row['DRB'],
+            season=row['season'],
-        TRB=row['TRB'],
+        )
-        AST=row['AST'],
+        player_data.save()
-        STL=row['STL'],
-        BLK=row['BLK'],
-        TOV=row['TOV'],
-        PF=row['PF'],
-        PTS=row['PTS'],
-        team=row['team'],
-        season=row['season'],
-    )
-    player_data.save()

nba_data/scraper_single.py

@@ -0,0 +1,102 @@
+import os
+import sys
+import django
+import requests
+from bs4 import BeautifulSoup
+import csv
+import json
+import time
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings')
+django.setup()
+
+from nba_data.models import PlayerData
+
+
+team = input('What team? ') 
+season = input('What season? ')
+
+
+url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html'
+response = requests.get(url_make)
+
+soup = BeautifulSoup(response.text, "html.parser")
+
+table = soup.find("table", {"id": "per_game"})
+
+if table is None:
+    print(f"Table not found for team {team} and season {season}.")
+    sys.exit(1)
+
+header_row = table.find("thead").find("tr")
+rows = table.find("tbody").find_all("tr")
+
+
+# Extract coloumn headers
+headers = [col.text.strip() for col in header_row.find_all("th")][1:]
+headers.extend(['team', 'season'])  # Add 'team' and 'season' fields to headers
+
+# Extract rows as dictionaries
+data = []
+for row in rows:
+    cols = row.find_all("td")
+    cols = [col.text.strip() for col in cols]
+    row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)}
+    row_dict['team'] = team
+    row_dict['season'] = season
+    data.append(row_dict)
+
+
+# locate current directory
+script_dir = os.path.dirname(os.path.abspath(__file__))
+
+# CSV file
+csv_output_path = os.path.join(script_dir, '..', 'data', 'output.csv')
+with open(csv_output_path, "w", newline="") as csvfile:
+    writer = csv.DictWriter(csvfile, fieldnames =headers)
+    writer.writeheader()
+    for row in data:
+        writer.writerow(row)
+
+# Save as JSON
+json_output_path = os.path.join(script_dir, '..', 'data', 'output.json')
+with open(json_output_path, 'w') as jsonfile:
+    json.dump(data, jsonfile)
+
+# save data to database
+for row in data:
+    player_data = PlayerData(
+        name=row['Player'],
+        age=row['Age'],
+        games=row['G'],
+        games_started=row['GS'],
+        minutes_pg=row['MP'],
+        field_goals=row['FG'],
+        field_attempts=row['FGA'],
+        field_percent=row['FG%'],
+        three_fg=row['3P'],
+        three_attempts=row['3PA'],
+        three_percent=row['3P%'],
+        two_fg=row['2P'],
+        two_attempts=row['2PA'],
+        two_percent=row['2P%'],
+        effect_fg_percent=row['eFG%'],
+        ft=row['FT'],
+        fta=row['FTA'],
+        ft_percent=row['FT%'],
+        ORB=row['ORB'],
+        DRB=row['DRB'],
+        TRB=row['TRB'],
+        AST=row['AST'],
+        STL=row['STL'],
+        BLK=row['BLK'],
+        TOV=row['TOV'],
+        PF=row['PF'],
+        PTS=row['PTS'],
+        team=row['team'],
+        season=row['season'],
+    )
+    player_data.save()

nba_stats/settings.py

@@ -11,6 +11,14 @@ https://docs.djangoproject.com/en/4.2/ref/settings/
 """
 
 from pathlib import Path
+from dotenv import load_dotenv
+load_dotenv()
+import os
+
+db_user = os.getenv('DB_USER')
+db_pass = os.getenv('DB_PASS')
+
+
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -76,8 +84,12 @@ WSGI_APPLICATION = 'nba_stats.wsgi.application'
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
+        'ENGINE': 'django.db.backends.postgresql',
-        'NAME': BASE_DIR / 'db.sqlite3',
+        'NAME': 'nba_stats',
+        'USER': db_user,
+        'PASSWORD': db_pass,
+        'HOST': 'localhost',
+        'PORT': '',
     }
 }