adjust model for outliers

nba_data/migrations/0010_alter_playeradvanceddata_per_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2 on 2023-05-01 21:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('nba_data', '0009_playerplayoffadvanceddata'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='playeradvanceddata',
+            name='PER',
+            field=models.DecimalField(decimal_places=1, max_digits=6, null=True),
+        ),
+        migrations.AlterField(
+            model_name='playerplayoffadvanceddata',
+            name='PER',
+            field=models.DecimalField(decimal_places=1, max_digits=6, null=True),
+        ),
+    ]

nba_data/migrations/0011_alter_playerplayoffadvanceddata_bpm_and_more.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2 on 2023-05-01 21:58
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('nba_data', '0010_alter_playeradvanceddata_per_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='playerplayoffadvanceddata',
+            name='bpm',
+            field=models.DecimalField(decimal_places=2, max_digits=5, null=True, validators=[django.core.validators.MinValueValidator(-100), django.core.validators.MaxValueValidator(100)]),
+        ),
+        migrations.AlterField(
+            model_name='playerplayoffadvanceddata',
+            name='dbpm',
+            field=models.DecimalField(decimal_places=2, max_digits=5, null=True, validators=[django.core.validators.MinValueValidator(-100), django.core.validators.MaxValueValidator(100)]),
+        ),
+        migrations.AlterField(
+            model_name='playerplayoffadvanceddata',
+            name='obpm',
+            field=models.DecimalField(decimal_places=2, max_digits=5, null=True, validators=[django.core.validators.MinValueValidator(-100), django.core.validators.MaxValueValidator(100)]),
+        ),
+    ]

nba_data/models.py

@@ -1,4 +1,5 @@
 from django.db import models
+from django.core.validators import MaxValueValidator, MinValueValidator
 
 # Create your models here.
 class PlayerData(models.Model):
@@ -75,7 +76,7 @@ class PlayerAdvancedData(models.Model):
     age = models.IntegerField(null=True)
     games = models.IntegerField(null=True)
     minutes_played = models.IntegerField(null=True)
-    PER = models.DecimalField(max_digits=5, decimal_places=1, null=True)
+    PER = models.DecimalField(max_digits=6, decimal_places=1, null=True)
     TS_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True)
     three_p_attempt_rate = models.DecimalField(max_digits=5, decimal_places=3, null=True)
     ft_attempt_rate = models.DecimalField(max_digits=5, decimal_places=3, null=True)
@@ -142,7 +143,7 @@ class PlayerPlayoffAdvancedData(models.Model):
     age = models.IntegerField(null=True)
     games = models.IntegerField(null=True)
     minutes_played = models.IntegerField(null=True)
-    PER = models.DecimalField(max_digits=5, decimal_places=1, null=True)
+    PER = models.DecimalField(max_digits=6, decimal_places=1, null=True)
     TS_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True)
     three_p_attempt_rate = models.DecimalField(max_digits=5, decimal_places=3, null=True)
     ft_attempt_rate = models.DecimalField(max_digits=5, decimal_places=3, null=True)
@@ -158,13 +159,43 @@ class PlayerPlayoffAdvancedData(models.Model):
     dws = models.DecimalField(max_digits=5, decimal_places=2, null=True)
     ws = models.DecimalField(max_digits=5, decimal_places=2, null=True)
     ws_per_48 = models.DecimalField(max_digits=5, decimal_places=3, null=True)
-    obpm = models.DecimalField(max_digits=5, decimal_places=2, null=True)
-    dbpm = models.DecimalField(max_digits=5, decimal_places=2, null=True)
-    bpm = models.DecimalField(max_digits=5, decimal_places=2, null=True)
+    obpm = models.DecimalField(max_digits=5, decimal_places=2, null=True, validators=[MinValueValidator(-100), MaxValueValidator(100)])
+    dbpm = models.DecimalField(max_digits=5, decimal_places=2, null=True, validators=[MinValueValidator(-100), MaxValueValidator(100)])
+    bpm = models.DecimalField(max_digits=5, decimal_places=2, null=True, validators=[MinValueValidator(-100), MaxValueValidator(100)])
     vorp = models.DecimalField(max_digits=5, decimal_places=2, null=True)
     team = models.CharField(max_length=30, default='')
     season = models.IntegerField(null=True)
     
+    def cap_extreme_values(self):
+        min_value = -100
+        max_value = 100
+        
+        obpm = float(self.obpm)
+        if obpm < min_value:
+            obpm = min_value
+        elif obpm > max_value:
+            obpm = max_value
+        self.obpm = obpm
+
+        dbpm = float(self.dbpm)
+        if dbpm < min_value:
+            dbpm = min_value
+        elif dbpm > max_value:
+            dbpm = max_value
+        self.dbpm = dbpm
+
+        bpm = float(self.bpm)
+        if bpm < min_value:
+            bpm = min_value
+        elif bpm > max_value:
+            bpm = max_value
+        self.bpm = bpm
+
+
+    def save(self, *args, **kwargs):
+        self.cap_extreme_values()
+        super(PlayerPlayoffAdvancedData, self).save(*args, **kwargs)
+    
     def __str__(self):
         return self.player_name
     

nba_data/nba_scraper_playoffs_advanced.py

@@ -0,0 +1,119 @@
+import os
+import sys
+import django
+import requests
+from bs4 import BeautifulSoup
+import csv
+import json
+import time
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings')
+django.setup()
+
+from nba_data.models import PlayerPlayoffAdvancedData
+
+# Change CHO to CHA 2014 and prior.
+# Change NOP to NOH for 2013 and prior.
+# Change OKC to SEA for 2008 and prior.
+# Change NOH to NOK for 2007 and 2006
+# Change NOK to NOH for 2005 and prior.
+# Change NOH to CHH for 2002 and prior. NOLA did not have a team before 2003.
+# Change WAS to WSB for 1997 and prior.
+# Change BRK to NJN for 2012 and prior.
+# Change MEM to VAN for 2001 and prior.
+# Change LAC to SDC for 1984 and prior.
+
+
+teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHA']
+
+team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS']
+
+season = input('What season? ')
+
+for team in teams:
+    url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html'
+    response = requests.get(url_make)
+
+    soup = BeautifulSoup(response.text, "html.parser")
+    
+    table = soup.find("table", {"id": "playoffs_advanced"})
+
+    if table is None:
+        print(f"Table not found for team {team} and season {season}.")
+        # sys.exit(1)
+        continue
+
+    header_row = table.find("thead").find("tr")
+    rows = table.find("tbody").find_all("tr")
+
+
+    # Extract coloumn headers
+    headers = [col.text.strip() for col in header_row.find_all("th")][1:]
+    headers.extend(['team', 'season'])  # Add 'team' and 'season' fields to headers
+
+    # Extract rows as dictionaries
+    data = []
+    for row in rows:
+        cols = row.find_all("td")
+        cols = [col.text.strip() for col in cols]
+        row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)}
+        row_dict['team'] = team
+        row_dict['season'] = season
+        data.append(row_dict)
+
+
+    # locate current directory
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    
+    # save as CSV file
+    csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_playoffs_advanced_output.csv')
+    with open(csv_output_path, "w", newline="") as csvfile:
+        writer = csv.DictWriter(csvfile, fieldnames=headers)
+        writer.writeheader()
+        for row in data:
+            writer.writerow(row)
+
+    # Save as JSON
+    json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_playoffs_advanced_output.json')
+    with open(json_output_path, 'w') as jsonfile:
+        json.dump(data, jsonfile)
+
+    # save data to database
+    for row in data:
+        player_data = PlayerPlayoffAdvancedData(
+            player_name = row['Player'],
+            age = row['Age'],
+            games = row['G'],
+            minutes_played = row['MP'],
+            PER = row['PER'],
+            TS_percent = row['TS%'],
+            three_p_attempt_rate = row['3PAr'],
+            ft_attempt_rate = row['FTr'],
+            orb_percent = row['ORB%'],
+            drb_percent = row['DRB%'],
+            trb_percent = row['TRB%'],
+            ast_percent = row['AST%'],
+            stl_percent = row['STL%'],
+            blk_percent = row['BLK%'],
+            tov_percent = row['TOV%'],
+            usg_percent = row['USG%'],
+            ows = row['OWS'],
+            dws = row['DWS'],
+            ws = row['WS'],
+            ws_per_48 = row['WS/48'],
+            obpm = row['OBPM'],
+            dbpm = row['DBPM'],
+            bpm = row['BPM'],
+            vorp = row['VORP'],
+            team = row['team'],
+            season = row['season'],
+        )
+        player_data.save()
+
+print('success')
+        
+
+    

nba_data/nba_scraper_playoffs_advanced_player.py

@@ -0,0 +1,127 @@
+import os
+import sys
+import django
+import requests
+from bs4 import BeautifulSoup
+import csv
+import json
+import time
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings')
+django.setup()
+
+from nba_data.models import PlayerPlayoffAdvancedData
+
+# Change CHO to CHA 2014 and prior.
+# Change NOP to NOH for 2013 and prior.
+# Change OKC to SEA for 2008 and prior.
+# Change NOH to NOK for 2007 and 2006
+# Change NOK to NOH for 2005 and prior.
+# Change NOH to CHH for 2002 and prior. NOLA did not have a team before 2003.
+# Change WAS to WSB for 1997 and prior.
+# Change BRK to NJN for 2012 and prior.
+# Change MEM to VAN for 2001 and prior.
+# Change LAC to SDC for 1984 and prior.
+
+
+teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHA']
+
+single_team = ['GSW']
+
+team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS']
+
+# change player name here
+target_player_name = 'Nemanja Nedović'
+
+season = input('What season? ')
+
+
+
+for team in single_team:
+    url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html'
+    response = requests.get(url_make)
+
+    soup = BeautifulSoup(response.text, "html.parser")
+    
+    table = soup.find("table", {"id": "playoffs_advanced"})
+
+    if table is None:
+        print(f"Table not found for team {team} and season {season}.")
+        # sys.exit(1)
+        continue
+
+    header_row = table.find("thead").find("tr")
+    rows = table.find("tbody").find_all("tr")
+
+
+    # Extract coloumn headers
+    headers = [col.text.strip() for col in header_row.find_all("th")][1:]
+    headers.extend(['team', 'season'])  # Add 'team' and 'season' fields to headers
+
+    # Extract rows as dictionaries
+    data = []
+    for row in rows:
+        cols = row.find_all("td")
+        cols = [col.text.strip() for col in cols]
+        row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)}
+        row_dict['team'] = team
+        row_dict['season'] = season
+        if row_dict['Player'] == target_player_name:
+            data.append(row_dict)
+
+
+    # locate current directory
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    
+    # save as CSV file
+    csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_playoffs_advanced_output.csv')
+    with open(csv_output_path, "w", newline="") as csvfile:
+        writer = csv.DictWriter(csvfile, fieldnames=headers)
+        writer.writeheader()
+        for row in data:
+            writer.writerow(row)
+
+    # Save as JSON
+    json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_playoffs_advanced_output.json')
+    with open(json_output_path, 'w') as jsonfile:
+        json.dump(data, jsonfile)
+
+    # save data to database
+    for row in data:
+        player_data = PlayerPlayoffAdvancedData(
+            player_name = row['Player'],
+            age = row['Age'],
+            games = row['G'],
+            minutes_played = row['MP'],
+            PER = row['PER'],
+            TS_percent = row['TS%'],
+            three_p_attempt_rate = row['3PAr'],
+            ft_attempt_rate = row['FTr'],
+            orb_percent = row['ORB%'],
+            drb_percent = row['DRB%'],
+            trb_percent = row['TRB%'],
+            ast_percent = row['AST%'],
+            stl_percent = row['STL%'],
+            blk_percent = row['BLK%'],
+            tov_percent = row['TOV%'],
+            usg_percent = row['USG%'],
+            ows = row['OWS'],
+            dws = row['DWS'],
+            ws = row['WS'],
+            ws_per_48 = row['WS/48'],
+            obpm = row['OBPM'],
+            dbpm = row['DBPM'],
+            bpm = row['BPM'],
+            vorp = row['VORP'],
+            team = row['team'],
+            season = row['season'],
+        )
+        player_data.save()
+
+print('success')
+        
+
+