From e51e25fe08ff8146bc5fa3add474cf67d63d9947 Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Mon, 1 May 2023 17:19:02 -0500 Subject: [PATCH] adjust model for outliers --- ...0_alter_playeradvanceddata_per_and_more.py | 23 ++++ ..._playerplayoffadvanceddata_bpm_and_more.py | 29 ++++ nba_data/models.py | 41 +++++- nba_data/nba_scraper_playoffs_advanced.py | 119 ++++++++++++++++ .../nba_scraper_playoffs_advanced_player.py | 127 ++++++++++++++++++ 5 files changed, 334 insertions(+), 5 deletions(-) create mode 100644 nba_data/migrations/0010_alter_playeradvanceddata_per_and_more.py create mode 100644 nba_data/migrations/0011_alter_playerplayoffadvanceddata_bpm_and_more.py create mode 100644 nba_data/nba_scraper_playoffs_advanced_player.py diff --git a/nba_data/migrations/0010_alter_playeradvanceddata_per_and_more.py b/nba_data/migrations/0010_alter_playeradvanceddata_per_and_more.py new file mode 100644 index 0000000..9cad619 --- /dev/null +++ b/nba_data/migrations/0010_alter_playeradvanceddata_per_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2 on 2023-05-01 21:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nba_data', '0009_playerplayoffadvanceddata'), + ] + + operations = [ + migrations.AlterField( + model_name='playeradvanceddata', + name='PER', + field=models.DecimalField(decimal_places=1, max_digits=6, null=True), + ), + migrations.AlterField( + model_name='playerplayoffadvanceddata', + name='PER', + field=models.DecimalField(decimal_places=1, max_digits=6, null=True), + ), + ] diff --git a/nba_data/migrations/0011_alter_playerplayoffadvanceddata_bpm_and_more.py b/nba_data/migrations/0011_alter_playerplayoffadvanceddata_bpm_and_more.py new file mode 100644 index 0000000..c230652 --- /dev/null +++ b/nba_data/migrations/0011_alter_playerplayoffadvanceddata_bpm_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2 on 2023-05-01 21:58 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nba_data', '0010_alter_playeradvanceddata_per_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='playerplayoffadvanceddata', + name='bpm', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True, validators=[django.core.validators.MinValueValidator(-100), django.core.validators.MaxValueValidator(100)]), + ), + migrations.AlterField( + model_name='playerplayoffadvanceddata', + name='dbpm', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True, validators=[django.core.validators.MinValueValidator(-100), django.core.validators.MaxValueValidator(100)]), + ), + migrations.AlterField( + model_name='playerplayoffadvanceddata', + name='obpm', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True, validators=[django.core.validators.MinValueValidator(-100), django.core.validators.MaxValueValidator(100)]), + ), + ] diff --git a/nba_data/models.py b/nba_data/models.py index ff6841c..f7c39ca 100644 --- a/nba_data/models.py +++ b/nba_data/models.py @@ -1,4 +1,5 @@ from django.db import models +from django.core.validators import MaxValueValidator, MinValueValidator # Create your models here. class PlayerData(models.Model): @@ -75,7 +76,7 @@ class PlayerAdvancedData(models.Model): age = models.IntegerField(null=True) games = models.IntegerField(null=True) minutes_played = models.IntegerField(null=True) - PER = models.DecimalField(max_digits=5, decimal_places=1, null=True) + PER = models.DecimalField(max_digits=6, decimal_places=1, null=True) TS_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) three_p_attempt_rate = models.DecimalField(max_digits=5, decimal_places=3, null=True) ft_attempt_rate = models.DecimalField(max_digits=5, decimal_places=3, null=True) @@ -142,7 +143,7 @@ class PlayerPlayoffAdvancedData(models.Model): age = models.IntegerField(null=True) games = models.IntegerField(null=True) minutes_played = models.IntegerField(null=True) - PER = models.DecimalField(max_digits=5, decimal_places=1, null=True) + PER = models.DecimalField(max_digits=6, decimal_places=1, null=True) TS_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) three_p_attempt_rate = models.DecimalField(max_digits=5, decimal_places=3, null=True) ft_attempt_rate = models.DecimalField(max_digits=5, decimal_places=3, null=True) @@ -158,13 +159,43 @@ class PlayerPlayoffAdvancedData(models.Model): dws = models.DecimalField(max_digits=5, decimal_places=2, null=True) ws = models.DecimalField(max_digits=5, decimal_places=2, null=True) ws_per_48 = models.DecimalField(max_digits=5, decimal_places=3, null=True) - obpm = models.DecimalField(max_digits=5, decimal_places=2, null=True) - dbpm = models.DecimalField(max_digits=5, decimal_places=2, null=True) - bpm = models.DecimalField(max_digits=5, decimal_places=2, null=True) + obpm = models.DecimalField(max_digits=5, decimal_places=2, null=True, validators=[MinValueValidator(-100), MaxValueValidator(100)]) + dbpm = models.DecimalField(max_digits=5, decimal_places=2, null=True, validators=[MinValueValidator(-100), MaxValueValidator(100)]) + bpm = models.DecimalField(max_digits=5, decimal_places=2, null=True, validators=[MinValueValidator(-100), MaxValueValidator(100)]) vorp = models.DecimalField(max_digits=5, decimal_places=2, null=True) team = models.CharField(max_length=30, default='') season = models.IntegerField(null=True) + def cap_extreme_values(self): + min_value = -100 + max_value = 100 + + obpm = float(self.obpm) + if obpm < min_value: + obpm = min_value + elif obpm > max_value: + obpm = max_value + self.obpm = obpm + + dbpm = float(self.dbpm) + if dbpm < min_value: + dbpm = min_value + elif dbpm > max_value: + dbpm = max_value + self.dbpm = dbpm + + bpm = float(self.bpm) + if bpm < min_value: + bpm = min_value + elif bpm > max_value: + bpm = max_value + self.bpm = bpm + + + def save(self, *args, **kwargs): + self.cap_extreme_values() + super(PlayerPlayoffAdvancedData, self).save(*args, **kwargs) + def __str__(self): return self.player_name diff --git a/nba_data/nba_scraper_playoffs_advanced.py b/nba_data/nba_scraper_playoffs_advanced.py index e69de29..77fee73 100644 --- a/nba_data/nba_scraper_playoffs_advanced.py +++ b/nba_data/nba_scraper_playoffs_advanced.py @@ -0,0 +1,119 @@ +import os +import sys +import django +import requests +from bs4 import BeautifulSoup +import csv +import json +import time + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings') +django.setup() + +from nba_data.models import PlayerPlayoffAdvancedData + +# Change CHO to CHA 2014 and prior. +# Change NOP to NOH for 2013 and prior. +# Change OKC to SEA for 2008 and prior. +# Change NOH to NOK for 2007 and 2006 +# Change NOK to NOH for 2005 and prior. +# Change NOH to CHH for 2002 and prior. NOLA did not have a team before 2003. +# Change WAS to WSB for 1997 and prior. +# Change BRK to NJN for 2012 and prior. +# Change MEM to VAN for 2001 and prior. +# Change LAC to SDC for 1984 and prior. + + +teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHA'] + +team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS'] + +season = input('What season? ') + +for team in teams: + url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html' + response = requests.get(url_make) + + soup = BeautifulSoup(response.text, "html.parser") + + table = soup.find("table", {"id": "playoffs_advanced"}) + + if table is None: + print(f"Table not found for team {team} and season {season}.") + # sys.exit(1) + continue + + header_row = table.find("thead").find("tr") + rows = table.find("tbody").find_all("tr") + + + # Extract coloumn headers + headers = [col.text.strip() for col in header_row.find_all("th")][1:] + headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers + + # Extract rows as dictionaries + data = [] + for row in rows: + cols = row.find_all("td") + cols = [col.text.strip() for col in cols] + row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)} + row_dict['team'] = team + row_dict['season'] = season + data.append(row_dict) + + + # locate current directory + script_dir = os.path.dirname(os.path.abspath(__file__)) + + # save as CSV file + csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_playoffs_advanced_output.csv') + with open(csv_output_path, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=headers) + writer.writeheader() + for row in data: + writer.writerow(row) + + # Save as JSON + json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_playoffs_advanced_output.json') + with open(json_output_path, 'w') as jsonfile: + json.dump(data, jsonfile) + + # save data to database + for row in data: + player_data = PlayerPlayoffAdvancedData( + player_name = row['Player'], + age = row['Age'], + games = row['G'], + minutes_played = row['MP'], + PER = row['PER'], + TS_percent = row['TS%'], + three_p_attempt_rate = row['3PAr'], + ft_attempt_rate = row['FTr'], + orb_percent = row['ORB%'], + drb_percent = row['DRB%'], + trb_percent = row['TRB%'], + ast_percent = row['AST%'], + stl_percent = row['STL%'], + blk_percent = row['BLK%'], + tov_percent = row['TOV%'], + usg_percent = row['USG%'], + ows = row['OWS'], + dws = row['DWS'], + ws = row['WS'], + ws_per_48 = row['WS/48'], + obpm = row['OBPM'], + dbpm = row['DBPM'], + bpm = row['BPM'], + vorp = row['VORP'], + team = row['team'], + season = row['season'], + ) + player_data.save() + +print('success') + + + \ No newline at end of file diff --git a/nba_data/nba_scraper_playoffs_advanced_player.py b/nba_data/nba_scraper_playoffs_advanced_player.py new file mode 100644 index 0000000..403ba3a --- /dev/null +++ b/nba_data/nba_scraper_playoffs_advanced_player.py @@ -0,0 +1,127 @@ +import os +import sys +import django +import requests +from bs4 import BeautifulSoup +import csv +import json +import time + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings') +django.setup() + +from nba_data.models import PlayerPlayoffAdvancedData + +# Change CHO to CHA 2014 and prior. +# Change NOP to NOH for 2013 and prior. +# Change OKC to SEA for 2008 and prior. +# Change NOH to NOK for 2007 and 2006 +# Change NOK to NOH for 2005 and prior. +# Change NOH to CHH for 2002 and prior. NOLA did not have a team before 2003. +# Change WAS to WSB for 1997 and prior. +# Change BRK to NJN for 2012 and prior. +# Change MEM to VAN for 2001 and prior. +# Change LAC to SDC for 1984 and prior. + + +teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHA'] + +single_team = ['GSW'] + +team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS'] + +# change player name here +target_player_name = 'Nemanja Nedović' + +season = input('What season? ') + + + +for team in single_team: + url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html' + response = requests.get(url_make) + + soup = BeautifulSoup(response.text, "html.parser") + + table = soup.find("table", {"id": "playoffs_advanced"}) + + if table is None: + print(f"Table not found for team {team} and season {season}.") + # sys.exit(1) + continue + + header_row = table.find("thead").find("tr") + rows = table.find("tbody").find_all("tr") + + + # Extract coloumn headers + headers = [col.text.strip() for col in header_row.find_all("th")][1:] + headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers + + # Extract rows as dictionaries + data = [] + for row in rows: + cols = row.find_all("td") + cols = [col.text.strip() for col in cols] + row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)} + row_dict['team'] = team + row_dict['season'] = season + if row_dict['Player'] == target_player_name: + data.append(row_dict) + + + # locate current directory + script_dir = os.path.dirname(os.path.abspath(__file__)) + + # save as CSV file + csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_playoffs_advanced_output.csv') + with open(csv_output_path, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=headers) + writer.writeheader() + for row in data: + writer.writerow(row) + + # Save as JSON + json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_playoffs_advanced_output.json') + with open(json_output_path, 'w') as jsonfile: + json.dump(data, jsonfile) + + # save data to database + for row in data: + player_data = PlayerPlayoffAdvancedData( + player_name = row['Player'], + age = row['Age'], + games = row['G'], + minutes_played = row['MP'], + PER = row['PER'], + TS_percent = row['TS%'], + three_p_attempt_rate = row['3PAr'], + ft_attempt_rate = row['FTr'], + orb_percent = row['ORB%'], + drb_percent = row['DRB%'], + trb_percent = row['TRB%'], + ast_percent = row['AST%'], + stl_percent = row['STL%'], + blk_percent = row['BLK%'], + tov_percent = row['TOV%'], + usg_percent = row['USG%'], + ows = row['OWS'], + dws = row['DWS'], + ws = row['WS'], + ws_per_48 = row['WS/48'], + obpm = row['OBPM'], + dbpm = row['DBPM'], + bpm = row['BPM'], + vorp = row['VORP'], + team = row['team'], + season = row['season'], + ) + player_data.save() + +print('success') + + + \ No newline at end of file