diff --git a/.gitignore b/.gitignore index cbe96eb..dfefb2b 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ data/ # OS generated files .DS_Store Thumbs.db + +# Environment Variables +.env diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9ee86e7 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "[python]": { + "editor.defaultFormatter": "ms-python.autopep8" + }, + "python.formatting.provider": "none" +} \ No newline at end of file diff --git a/chromedriver b/chromedriver new file mode 100755 index 0000000..caf7e4b Binary files /dev/null and b/chromedriver differ diff --git a/db.sqlite3 b/db.sqlite3 index 53c32b8..fe73b06 100644 Binary files a/db.sqlite3 and b/db.sqlite3 differ diff --git a/db.sqlite3-journal b/db.sqlite3-journal deleted file mode 100644 index b6c4cc6..0000000 Binary files a/db.sqlite3-journal and /dev/null differ diff --git a/nba_data/nba_verify.py b/nba_data/nba_verify.py new file mode 100644 index 0000000..e69de29 diff --git a/nba_data/scraper.py b/nba_data/scraper.py index b329f4d..2d510b6 100644 --- a/nba_data/scraper.py +++ b/nba_data/scraper.py @@ -5,6 +5,7 @@ import requests from bs4 import BeautifulSoup import csv import json +import time sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -14,85 +15,91 @@ django.setup() from nba_data.models import PlayerData -team = input('What team? ') +teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHO'] +team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS'] + +# team = input('What team? ') +# OMIT above. Will now only ask for season and iterate thru teams list. season = input('What season? ') -url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html' +for team in teams: + url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html' + response = requests.get(url_make) -url = 'https://www.basketball-reference.com/teams/HOU/2018.html' -lakers = 'https://www.basketball-reference.com/teams/LAL/2020.html' + soup = BeautifulSoup(response.text, "html.parser") + + table = soup.find("table", {"id": "per_game"}) -response = requests.get(url_make) + if table is None: + print(f"Table not found for team {team} and season {season}.") + sys.exit(1) -soup = BeautifulSoup(response.text, "html.parser") - -table = soup.find("table", {"id": "per_game"}) -# rows = table.find_all("tr") -header_row = table.find("thead").find("tr") -rows = table.find("tbody").find_all("tr") - -# Extract coloumn headers -headers = [col.text.strip() for col in header_row.find_all("th")][1:] -headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers - -# Extract rows as dictionaries -data = [] -for row in rows: - cols = row.find_all("td") - cols = [col.text.strip() for col in cols] - row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)} - row_dict['team'] = team - row_dict['season'] = season - data.append(row_dict) + header_row = table.find("thead").find("tr") + rows = table.find("tbody").find_all("tr") -# locate current directory -script_dir = os.path.dirname(os.path.abspath(__file__)) + # Extract coloumn headers + headers = [col.text.strip() for col in header_row.find_all("th")][1:] + headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers -# CSV file -csv_output_path = os.path.join(script_dir, '..', 'data', 'output.csv') -with open(csv_output_path, "w", newline="") as csvfile: - writer = csv.DictWriter(csvfile, fieldnames =headers) - writer.writeheader() + # Extract rows as dictionaries + data = [] + for row in rows: + cols = row.find_all("td") + cols = [col.text.strip() for col in cols] + row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)} + row_dict['team'] = team + row_dict['season'] = season + data.append(row_dict) + + + # locate current directory + script_dir = os.path.dirname(os.path.abspath(__file__)) + + # CSV file + csv_output_path = os.path.join(script_dir, '..', 'data', 'output.csv') + with open(csv_output_path, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames =headers) + writer.writeheader() + for row in data: + writer.writerow(row) + + # Save as JSON + json_output_path = os.path.join(script_dir, '..', 'data', 'output.json') + with open(json_output_path, 'w') as jsonfile: + json.dump(data, jsonfile) + + # save data to database for row in data: - writer.writerow(row) - -# Save as JSON -json_output_path = os.path.join(script_dir, '..', 'data', 'output.json') -with open(json_output_path, 'w') as jsonfile: - json.dump(data, jsonfile) - -# save data to database -for row in data: - player_data = PlayerData( - name=row['Player'], - age=row['Age'], - games=row['G'], - games_started=row['GS'], - minutes_pg=row['MP'], - field_goals=row['FG'], - field_attempts=row['FGA'], - field_percent=row['FG%'], - three_fg=row['3P'], - three_attempts=row['3PA'], - three_percent=row['3P%'], - two_fg=row['2P'], - two_attempts=row['2PA'], - two_percent=row['2P%'], - effect_fg_percent=row['eFG%'], - ft=row['FT'], - fta=row['FTA'], - ft_percent=row['FT%'], - ORB=row['ORB'], - DRB=row['DRB'], - TRB=row['TRB'], - AST=row['AST'], - STL=row['STL'], - BLK=row['BLK'], - TOV=row['TOV'], - PF=row['PF'], - PTS=row['PTS'], - team=row['team'], - season=row['season'], - ) - player_data.save() + player_data = PlayerData( + name=row['Player'], + age=row['Age'], + games=row['G'], + games_started=row['GS'], + minutes_pg=row['MP'], + field_goals=row['FG'], + field_attempts=row['FGA'], + field_percent=row['FG%'], + three_fg=row['3P'], + three_attempts=row['3PA'], + three_percent=row['3P%'], + two_fg=row['2P'], + two_attempts=row['2PA'], + two_percent=row['2P%'], + effect_fg_percent=row['eFG%'], + ft=row['FT'], + fta=row['FTA'], + ft_percent=row['FT%'], + ORB=row['ORB'], + DRB=row['DRB'], + TRB=row['TRB'], + AST=row['AST'], + STL=row['STL'], + BLK=row['BLK'], + TOV=row['TOV'], + PF=row['PF'], + PTS=row['PTS'], + team=row['team'], + season=row['season'], + ) + player_data.save() \ No newline at end of file diff --git a/nba_data/scraper_single.py b/nba_data/scraper_single.py new file mode 100644 index 0000000..05e7f96 --- /dev/null +++ b/nba_data/scraper_single.py @@ -0,0 +1,102 @@ +import os +import sys +import django +import requests +from bs4 import BeautifulSoup +import csv +import json +import time + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings') +django.setup() + +from nba_data.models import PlayerData + + +team = input('What team? ') +season = input('What season? ') + + +url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html' +response = requests.get(url_make) + +soup = BeautifulSoup(response.text, "html.parser") + +table = soup.find("table", {"id": "per_game"}) + +if table is None: + print(f"Table not found for team {team} and season {season}.") + sys.exit(1) + +header_row = table.find("thead").find("tr") +rows = table.find("tbody").find_all("tr") + + +# Extract coloumn headers +headers = [col.text.strip() for col in header_row.find_all("th")][1:] +headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers + +# Extract rows as dictionaries +data = [] +for row in rows: + cols = row.find_all("td") + cols = [col.text.strip() for col in cols] + row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)} + row_dict['team'] = team + row_dict['season'] = season + data.append(row_dict) + + +# locate current directory +script_dir = os.path.dirname(os.path.abspath(__file__)) + +# CSV file +csv_output_path = os.path.join(script_dir, '..', 'data', 'output.csv') +with open(csv_output_path, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames =headers) + writer.writeheader() + for row in data: + writer.writerow(row) + +# Save as JSON +json_output_path = os.path.join(script_dir, '..', 'data', 'output.json') +with open(json_output_path, 'w') as jsonfile: + json.dump(data, jsonfile) + +# save data to database +for row in data: + player_data = PlayerData( + name=row['Player'], + age=row['Age'], + games=row['G'], + games_started=row['GS'], + minutes_pg=row['MP'], + field_goals=row['FG'], + field_attempts=row['FGA'], + field_percent=row['FG%'], + three_fg=row['3P'], + three_attempts=row['3PA'], + three_percent=row['3P%'], + two_fg=row['2P'], + two_attempts=row['2PA'], + two_percent=row['2P%'], + effect_fg_percent=row['eFG%'], + ft=row['FT'], + fta=row['FTA'], + ft_percent=row['FT%'], + ORB=row['ORB'], + DRB=row['DRB'], + TRB=row['TRB'], + AST=row['AST'], + STL=row['STL'], + BLK=row['BLK'], + TOV=row['TOV'], + PF=row['PF'], + PTS=row['PTS'], + team=row['team'], + season=row['season'], + ) + player_data.save() \ No newline at end of file diff --git a/nba_stats/settings.py b/nba_stats/settings.py index 128942d..6ceb398 100644 --- a/nba_stats/settings.py +++ b/nba_stats/settings.py @@ -11,6 +11,14 @@ https://docs.djangoproject.com/en/4.2/ref/settings/ """ from pathlib import Path +from dotenv import load_dotenv +load_dotenv() +import os + +db_user = os.getenv('DB_USER') +db_pass = os.getenv('DB_PASS') + + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -76,8 +84,12 @@ WSGI_APPLICATION = 'nba_stats.wsgi.application' DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': BASE_DIR / 'db.sqlite3', + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'nba_stats', + 'USER': db_user, + 'PASSWORD': db_pass, + 'HOST': 'localhost', + 'PORT': '', } }