diff --git a/nba_data/migrations/0006_playeradvanceddata_playertotalsdata.py b/nba_data/migrations/0006_playeradvanceddata_playertotalsdata.py new file mode 100644 index 0000000..d0cfcd3 --- /dev/null +++ b/nba_data/migrations/0006_playeradvanceddata_playertotalsdata.py @@ -0,0 +1,80 @@ +# Generated by Django 4.2 on 2023-04-27 18:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nba_data', '0005_alter_playerdata_ast_alter_playerdata_blk_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='PlayerAdvancedData', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('player_name', models.CharField(max_length=255, null=True)), + ('age', models.IntegerField(null=True)), + ('games', models.IntegerField(null=True)), + ('minutes_played', models.IntegerField(null=True)), + ('PER', models.DecimalField(decimal_places=1, max_digits=5, null=True)), + ('TS_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('three_p_attempt_rate', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('ft_attempt_rate', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('orb_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('drb_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('trb_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('ast_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('stl_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('blk_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('tov_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('usg_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('ows', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('dws', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('ws', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('ws_per_48', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('obpm', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('dbpm', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('bpm', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('vorp', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('team', models.CharField(default='', max_length=30)), + ('season', models.IntegerField(null=True)), + ], + ), + migrations.CreateModel( + name='PlayerTotalsData', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('player_name', models.CharField(max_length=255, null=True)), + ('age', models.IntegerField(null=True)), + ('games', models.IntegerField(null=True)), + ('games_started', models.IntegerField(null=True)), + ('minutes_played', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('field_goals', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('field_attempts', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('field_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('three_fg', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('three_attempts', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('three_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('two_fg', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('two_attempts', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('two_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('effect_fg_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('ft', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('fta', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('ft_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('ORB', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('DRB', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('TRB', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('AST', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('STL', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('BLK', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('TOV', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('PF', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('PTS', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('team', models.CharField(default='', max_length=30)), + ('season', models.IntegerField(null=True)), + ], + ), + ] diff --git a/nba_data/models.py b/nba_data/models.py index 44b7c4e..5a1fc43 100644 --- a/nba_data/models.py +++ b/nba_data/models.py @@ -95,6 +95,8 @@ class PlayerAdvancedData(models.Model): dbpm = models.DecimalField(max_digits=5, decimal_places=2, null=True) bpm = models.DecimalField(max_digits=5, decimal_places=2, null=True) vorp = models.DecimalField(max_digits=5, decimal_places=2, null=True) + team = models.CharField(max_length=30, default='') + season = models.IntegerField(null=True) def __str__(self): return self.player_name diff --git a/nba_data/nba_scraper_advanced.py b/nba_data/nba_scraper_advanced.py new file mode 100644 index 0000000..97eaf93 --- /dev/null +++ b/nba_data/nba_scraper_advanced.py @@ -0,0 +1,109 @@ +import os +import sys +import django +import requests +from bs4 import BeautifulSoup +import csv +import json +import time + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings') +django.setup() + +from nba_data.models import PlayerAdvancedData + +teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHO'] +team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS'] + +# team = input('What team? ') +# OMIT above. Will now only ask for season and iterate thru teams list. +season = input('What season? ') + +for team in teams: + url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html' + response = requests.get(url_make) + + soup = BeautifulSoup(response.text, "html.parser") + + table = soup.find("table", {"id": "advanced"}) + + if table is None: + print(f"Table not found for team {team} and season {season}.") + # sys.exit(1) + continue + + header_row = table.find("thead").find("tr") + rows = table.find("tbody").find_all("tr") + + + # Extract coloumn headers + headers = [col.text.strip() for col in header_row.find_all("th")][1:] + headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers + + # Extract rows as dictionaries + data = [] + for row in rows: + cols = row.find_all("td") + cols = [col.text.strip() for col in cols] + row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)} + row_dict['team'] = team + row_dict['season'] = season + data.append(row_dict) + + + # locate current directory + script_dir = os.path.dirname(os.path.abspath(__file__)) + + # save as CSV file + csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_advanced_output.csv') + with open(csv_output_path, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=headers) + writer.writeheader() + for row in data: + writer.writerow(row) + + # Save as JSON + json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_advanced_output.json') + with open(json_output_path, 'w') as jsonfile: + json.dump(data, jsonfile) + + # # save data to database + # for row in data: + # player_data = PlayerData( + # name=row['Player'], + # age=row['Age'], + # games=row['G'], + # games_started=row['GS'], + # minutes_pg=row['MP'], + # field_goals=row['FG'], + # field_attempts=row['FGA'], + # field_percent=row['FG%'], + # three_fg=row['3P'], + # three_attempts=row['3PA'], + # three_percent=row['3P%'], + # two_fg=row['2P'], + # two_attempts=row['2PA'], + # two_percent=row['2P%'], + # effect_fg_percent=row['eFG%'], + # ft=row['FT'], + # fta=row['FTA'], + # ft_percent=row['FT%'], + # ORB=row['ORB'], + # DRB=row['DRB'], + # TRB=row['TRB'], + # AST=row['AST'], + # STL=row['STL'], + # BLK=row['BLK'], + # TOV=row['TOV'], + # PF=row['PF'], + # PTS=row['PTS'], + # team=row['team'], + # season=row['season'], + # ) + # player_data.save() + + + \ No newline at end of file diff --git a/nba_data/nba_scraper_totals.py b/nba_data/nba_scraper_totals.py new file mode 100644 index 0000000..7cf2561 --- /dev/null +++ b/nba_data/nba_scraper_totals.py @@ -0,0 +1,109 @@ +import os +import sys +import django +import requests +from bs4 import BeautifulSoup +import csv +import json +import time + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings') +django.setup() + +from nba_data.models import PlayerTotalsData + +teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHO'] +team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS'] + +# team = input('What team? ') +# OMIT above. Will now only ask for season and iterate thru teams list. +season = input('What season? ') + +for team in teams: + url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html' + response = requests.get(url_make) + + soup = BeautifulSoup(response.text, "html.parser") + + table = soup.find("table", {"id": "totals"}) + + if table is None: + print(f"Table not found for team {team} and season {season}.") + # sys.exit(1) + continue + + header_row = table.find("thead").find("tr") + rows = table.find("tbody").find_all("tr") + + + # Extract coloumn headers + headers = [col.text.strip() for col in header_row.find_all("th")][1:] + headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers + + # Extract rows as dictionaries + data = [] + for row in rows: + cols = row.find_all("td") + cols = [col.text.strip() for col in cols] + row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)} + row_dict['team'] = team + row_dict['season'] = season + data.append(row_dict) + + + # locate current directory + script_dir = os.path.dirname(os.path.abspath(__file__)) + + # save as CSV file + csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_totals_output.csv') + with open(csv_output_path, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=headers) + writer.writeheader() + for row in data: + writer.writerow(row) + + # Save as JSON + json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_totals_output.json') + with open(json_output_path, 'w') as jsonfile: + json.dump(data, jsonfile) + + # # save data to database + # for row in data: + # player_data = PlayerData( + # name=row['Player'], + # age=row['Age'], + # games=row['G'], + # games_started=row['GS'], + # minutes_pg=row['MP'], + # field_goals=row['FG'], + # field_attempts=row['FGA'], + # field_percent=row['FG%'], + # three_fg=row['3P'], + # three_attempts=row['3PA'], + # three_percent=row['3P%'], + # two_fg=row['2P'], + # two_attempts=row['2PA'], + # two_percent=row['2P%'], + # effect_fg_percent=row['eFG%'], + # ft=row['FT'], + # fta=row['FTA'], + # ft_percent=row['FT%'], + # ORB=row['ORB'], + # DRB=row['DRB'], + # TRB=row['TRB'], + # AST=row['AST'], + # STL=row['STL'], + # BLK=row['BLK'], + # TOV=row['TOV'], + # PF=row['PF'], + # PTS=row['PTS'], + # team=row['team'], + # season=row['season'], + # ) + # player_data.save() + + + \ No newline at end of file