From abe22c73f1a83aa728a2614ac03701a0915f08e4 Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:09:34 -0500 Subject: [PATCH 1/3] create player total data model --- nba_data/models.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/nba_data/models.py b/nba_data/models.py index d555929..4baedf1 100644 --- a/nba_data/models.py +++ b/nba_data/models.py @@ -34,4 +34,36 @@ class PlayerData(models.Model): def __str__(self): return self.name + +class PlayerTotalsData(models.Model): + player_name = models.CharField(max_length=255, null=True) + age = models.IntegerField(null=True) + games = models.IntegerField(null=True) + games_started = models.IntegerField(null=True) + minutes_played = models.DecimalField(max_digits=5, decimal_places=2, null=True) + field_goals = models.DecimalField(max_digits=5, decimal_places=2, null=True) + field_attempts = models.DecimalField(max_digits=5, decimal_places=2, null=True) + field_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) + three_fg = models.DecimalField(max_digits=5, decimal_places=2, null=True) + three_attempts = models.DecimalField(max_digits=5, decimal_places=2, null=True) + three_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) + two_fg = models.DecimalField(max_digits=5, decimal_places=2, null=True) + two_attempts = models.DecimalField(max_digits=5, decimal_places=2, null=True) + two_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) + effect_fg_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) + ft = models.DecimalField(max_digits=5, decimal_places=2, null=True) + fta = models.DecimalField(max_digits=5, decimal_places=2, null=True) + ft_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) + ORB= models.DecimalField(max_digits=5, decimal_places=2, null=True) + DRB = models.DecimalField(max_digits=5, decimal_places=2, null=True) + TRB = models.DecimalField(max_digits=5, decimal_places=2, null=True) + AST = models.DecimalField(max_digits=5, decimal_places=2, null=True) + STL = models.DecimalField(max_digits=5, decimal_places=2, null=True) + BLK = models.DecimalField(max_digits=5, decimal_places=2, null=True) + TOV = models.DecimalField(max_digits=5, decimal_places=2, null=True) + PF = models.DecimalField(max_digits=5, decimal_places=2, null=True) + PTS = models.DecimalField(max_digits=5, decimal_places=2, null=True) + team = models.CharField(max_length=30, default='') + season = models.IntegerField(null=True) + From d8f02a331be38e2cd70c4920c37b9ad52a396a5b Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:24:40 -0500 Subject: [PATCH 2/3] advanced model added --- nba_data/models.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/nba_data/models.py b/nba_data/models.py index 4baedf1..44b7c4e 100644 --- a/nba_data/models.py +++ b/nba_data/models.py @@ -47,7 +47,7 @@ class PlayerTotalsData(models.Model): three_fg = models.DecimalField(max_digits=5, decimal_places=2, null=True) three_attempts = models.DecimalField(max_digits=5, decimal_places=2, null=True) three_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) - two_fg = models.DecimalField(max_digits=5, decimal_places=2, null=True) + two_fg = models.DecimalField(max_digits=5, decimal_places=3, null=True) two_attempts = models.DecimalField(max_digits=5, decimal_places=2, null=True) two_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) effect_fg_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) @@ -66,4 +66,41 @@ class PlayerTotalsData(models.Model): team = models.CharField(max_length=30, default='') season = models.IntegerField(null=True) + def __str__(self): + return self.player_name + + +class PlayerAdvancedData(models.Model): + player_name = models.CharField(max_length=255, null=True) + age = models.IntegerField(null=True) + games = models.IntegerField(null=True) + minutes_played = models.IntegerField(null=True) + PER = models.DecimalField(max_digits=5, decimal_places=1, null=True) + TS_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) + three_p_attempt_rate = models.DecimalField(max_digits=5, decimal_places=3, null=True) + ft_attempt_rate = models.DecimalField(max_digits=5, decimal_places=3, null=True) + orb_percent = models.DecimalField(max_digits=5, decimal_places=2, null=True) + drb_percent = models.DecimalField(max_digits=5, decimal_places=2, null=True) + trb_percent = models.DecimalField(max_digits=5, decimal_places=2, null=True) + ast_percent = models.DecimalField(max_digits=5, decimal_places=2, null=True) + stl_percent = models.DecimalField(max_digits=5, decimal_places=2, null=True) + blk_percent = models.DecimalField(max_digits=5, decimal_places=2, null=True) + tov_percent = models.DecimalField(max_digits=5, decimal_places=2, null=True) + usg_percent = models.DecimalField(max_digits=5, decimal_places=2, null=True) + ows = models.DecimalField(max_digits=5, decimal_places=2, null=True) + dws = models.DecimalField(max_digits=5, decimal_places=2, null=True) + ws = models.DecimalField(max_digits=5, decimal_places=2, null=True) + ws_per_48 = models.DecimalField(max_digits=5, decimal_places=3, null=True) + obpm = models.DecimalField(max_digits=5, decimal_places=2, null=True) + dbpm = models.DecimalField(max_digits=5, decimal_places=2, null=True) + bpm = models.DecimalField(max_digits=5, decimal_places=2, null=True) + vorp = models.DecimalField(max_digits=5, decimal_places=2, null=True) + + def __str__(self): + return self.player_name + + + + + From 9efa5685c8bca52b2e637e7e4e4359d043605e5d Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:41:11 -0500 Subject: [PATCH 3/3] added and tested scraper for totals and advanced data --- ...006_playeradvanceddata_playertotalsdata.py | 80 +++++++++++++ nba_data/models.py | 2 + nba_data/nba_scraper_advanced.py | 109 ++++++++++++++++++ nba_data/nba_scraper_totals.py | 109 ++++++++++++++++++ 4 files changed, 300 insertions(+) create mode 100644 nba_data/migrations/0006_playeradvanceddata_playertotalsdata.py create mode 100644 nba_data/nba_scraper_advanced.py create mode 100644 nba_data/nba_scraper_totals.py diff --git a/nba_data/migrations/0006_playeradvanceddata_playertotalsdata.py b/nba_data/migrations/0006_playeradvanceddata_playertotalsdata.py new file mode 100644 index 0000000..d0cfcd3 --- /dev/null +++ b/nba_data/migrations/0006_playeradvanceddata_playertotalsdata.py @@ -0,0 +1,80 @@ +# Generated by Django 4.2 on 2023-04-27 18:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nba_data', '0005_alter_playerdata_ast_alter_playerdata_blk_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='PlayerAdvancedData', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('player_name', models.CharField(max_length=255, null=True)), + ('age', models.IntegerField(null=True)), + ('games', models.IntegerField(null=True)), + ('minutes_played', models.IntegerField(null=True)), + ('PER', models.DecimalField(decimal_places=1, max_digits=5, null=True)), + ('TS_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('three_p_attempt_rate', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('ft_attempt_rate', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('orb_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('drb_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('trb_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('ast_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('stl_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('blk_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('tov_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('usg_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('ows', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('dws', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('ws', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('ws_per_48', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('obpm', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('dbpm', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('bpm', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('vorp', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('team', models.CharField(default='', max_length=30)), + ('season', models.IntegerField(null=True)), + ], + ), + migrations.CreateModel( + name='PlayerTotalsData', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('player_name', models.CharField(max_length=255, null=True)), + ('age', models.IntegerField(null=True)), + ('games', models.IntegerField(null=True)), + ('games_started', models.IntegerField(null=True)), + ('minutes_played', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('field_goals', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('field_attempts', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('field_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('three_fg', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('three_attempts', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('three_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('two_fg', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('two_attempts', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('two_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('effect_fg_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('ft', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('fta', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('ft_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)), + ('ORB', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('DRB', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('TRB', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('AST', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('STL', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('BLK', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('TOV', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('PF', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('PTS', models.DecimalField(decimal_places=2, max_digits=5, null=True)), + ('team', models.CharField(default='', max_length=30)), + ('season', models.IntegerField(null=True)), + ], + ), + ] diff --git a/nba_data/models.py b/nba_data/models.py index 44b7c4e..5a1fc43 100644 --- a/nba_data/models.py +++ b/nba_data/models.py @@ -95,6 +95,8 @@ class PlayerAdvancedData(models.Model): dbpm = models.DecimalField(max_digits=5, decimal_places=2, null=True) bpm = models.DecimalField(max_digits=5, decimal_places=2, null=True) vorp = models.DecimalField(max_digits=5, decimal_places=2, null=True) + team = models.CharField(max_length=30, default='') + season = models.IntegerField(null=True) def __str__(self): return self.player_name diff --git a/nba_data/nba_scraper_advanced.py b/nba_data/nba_scraper_advanced.py new file mode 100644 index 0000000..97eaf93 --- /dev/null +++ b/nba_data/nba_scraper_advanced.py @@ -0,0 +1,109 @@ +import os +import sys +import django +import requests +from bs4 import BeautifulSoup +import csv +import json +import time + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings') +django.setup() + +from nba_data.models import PlayerAdvancedData + +teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHO'] +team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS'] + +# team = input('What team? ') +# OMIT above. Will now only ask for season and iterate thru teams list. +season = input('What season? ') + +for team in teams: + url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html' + response = requests.get(url_make) + + soup = BeautifulSoup(response.text, "html.parser") + + table = soup.find("table", {"id": "advanced"}) + + if table is None: + print(f"Table not found for team {team} and season {season}.") + # sys.exit(1) + continue + + header_row = table.find("thead").find("tr") + rows = table.find("tbody").find_all("tr") + + + # Extract coloumn headers + headers = [col.text.strip() for col in header_row.find_all("th")][1:] + headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers + + # Extract rows as dictionaries + data = [] + for row in rows: + cols = row.find_all("td") + cols = [col.text.strip() for col in cols] + row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)} + row_dict['team'] = team + row_dict['season'] = season + data.append(row_dict) + + + # locate current directory + script_dir = os.path.dirname(os.path.abspath(__file__)) + + # save as CSV file + csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_advanced_output.csv') + with open(csv_output_path, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=headers) + writer.writeheader() + for row in data: + writer.writerow(row) + + # Save as JSON + json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_advanced_output.json') + with open(json_output_path, 'w') as jsonfile: + json.dump(data, jsonfile) + + # # save data to database + # for row in data: + # player_data = PlayerData( + # name=row['Player'], + # age=row['Age'], + # games=row['G'], + # games_started=row['GS'], + # minutes_pg=row['MP'], + # field_goals=row['FG'], + # field_attempts=row['FGA'], + # field_percent=row['FG%'], + # three_fg=row['3P'], + # three_attempts=row['3PA'], + # three_percent=row['3P%'], + # two_fg=row['2P'], + # two_attempts=row['2PA'], + # two_percent=row['2P%'], + # effect_fg_percent=row['eFG%'], + # ft=row['FT'], + # fta=row['FTA'], + # ft_percent=row['FT%'], + # ORB=row['ORB'], + # DRB=row['DRB'], + # TRB=row['TRB'], + # AST=row['AST'], + # STL=row['STL'], + # BLK=row['BLK'], + # TOV=row['TOV'], + # PF=row['PF'], + # PTS=row['PTS'], + # team=row['team'], + # season=row['season'], + # ) + # player_data.save() + + + \ No newline at end of file diff --git a/nba_data/nba_scraper_totals.py b/nba_data/nba_scraper_totals.py new file mode 100644 index 0000000..7cf2561 --- /dev/null +++ b/nba_data/nba_scraper_totals.py @@ -0,0 +1,109 @@ +import os +import sys +import django +import requests +from bs4 import BeautifulSoup +import csv +import json +import time + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings') +django.setup() + +from nba_data.models import PlayerTotalsData + +teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHO'] +team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS'] + +# team = input('What team? ') +# OMIT above. Will now only ask for season and iterate thru teams list. +season = input('What season? ') + +for team in teams: + url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html' + response = requests.get(url_make) + + soup = BeautifulSoup(response.text, "html.parser") + + table = soup.find("table", {"id": "totals"}) + + if table is None: + print(f"Table not found for team {team} and season {season}.") + # sys.exit(1) + continue + + header_row = table.find("thead").find("tr") + rows = table.find("tbody").find_all("tr") + + + # Extract coloumn headers + headers = [col.text.strip() for col in header_row.find_all("th")][1:] + headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers + + # Extract rows as dictionaries + data = [] + for row in rows: + cols = row.find_all("td") + cols = [col.text.strip() for col in cols] + row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)} + row_dict['team'] = team + row_dict['season'] = season + data.append(row_dict) + + + # locate current directory + script_dir = os.path.dirname(os.path.abspath(__file__)) + + # save as CSV file + csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_totals_output.csv') + with open(csv_output_path, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=headers) + writer.writeheader() + for row in data: + writer.writerow(row) + + # Save as JSON + json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_totals_output.json') + with open(json_output_path, 'w') as jsonfile: + json.dump(data, jsonfile) + + # # save data to database + # for row in data: + # player_data = PlayerData( + # name=row['Player'], + # age=row['Age'], + # games=row['G'], + # games_started=row['GS'], + # minutes_pg=row['MP'], + # field_goals=row['FG'], + # field_attempts=row['FGA'], + # field_percent=row['FG%'], + # three_fg=row['3P'], + # three_attempts=row['3PA'], + # three_percent=row['3P%'], + # two_fg=row['2P'], + # two_attempts=row['2PA'], + # two_percent=row['2P%'], + # effect_fg_percent=row['eFG%'], + # ft=row['FT'], + # fta=row['FTA'], + # ft_percent=row['FT%'], + # ORB=row['ORB'], + # DRB=row['DRB'], + # TRB=row['TRB'], + # AST=row['AST'], + # STL=row['STL'], + # BLK=row['BLK'], + # TOV=row['TOV'], + # PF=row['PF'], + # PTS=row['PTS'], + # team=row['team'], + # season=row['season'], + # ) + # player_data.save() + + + \ No newline at end of file