salary scraper success;

nba_data/migrations/0013_playersalarydata_season_playersalarydata_team.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2 on 2023-05-09 00:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('nba_data', '0012_playersalarydata'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='playersalarydata',
+            name='season',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name='playersalarydata',
+            name='team',
+            field=models.CharField(default='', max_length=30),
+        ),
+    ]

nba_data/models.py

@@ -203,6 +203,8 @@ class PlayerPlayoffAdvancedData(models.Model):
 class PlayerSalaryData(models.Model):
     player_name = models.CharField(max_length=255, null=True)
     salary = models.IntegerField(null=True)
+    team = models.CharField(max_length=30, default='')
+    season = models.IntegerField(null=True)
     
     def __str__(self):
         return self.player_name

nba_data/nba_scraper_playoffs_totals.py

@@ -29,6 +29,7 @@ from nba_data.models import PlayerPlayoffTotalsData
 
 teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'NJN', 'TOR', 'MEM', 'NOH', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHA']
 
+
 team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS']
 
 season = input('What season? ')

nba_data/nba_scraper_salary.py

@@ -0,0 +1,104 @@
+import os
+import sys
+import django
+import requests
+from bs4 import BeautifulSoup, Comment
+import csv
+import json
+import time
+import re
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings')
+django.setup()
+
+from nba_data.models import PlayerSalaryData
+
+# Change CHO to CHA 2014 and prior.
+# Change NOP to NOH for 2013 and prior.
+# Change OKC to SEA for 2008 and prior.
+# Change NOH to NOK for 2007 and 2006
+# Change NOK to NOH for 2005 and prior.
+# Change NOH to CHH for 2002 and prior. NOLA did not have a team before 2003.
+# Change WAS to WSB for 1997 and prior.
+# Change BRK to NJN for 2012 and prior.
+# Change MEM to VAN for 2001 and prior.
+# Change LAC to SDC for 1984 and prior.
+
+
+teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHO']
+teams_test = ['HOU', 'PHI', 'BOS']
+
+season = input('What season? ')
+
+for team in teams_test:
+    url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html'
+    response = requests.get(url_make)
+    
+    soup = BeautifulSoup(response.text, "html.parser")
+
+    comment = soup.find(string=lambda text: isinstance(text, Comment) and "salaries2" in text)
+    table_soup = BeautifulSoup(comment, "html.parser")
+    table = table_soup.find("table", {'class': 'suppress_all stats_table', 'id': 'salaries2'})
+
+    if table is None:
+        print(f"Table not found for team {team} and season {season}.")
+        # sys.exit(1)
+        continue
+
+    header_row = table.find("thead").find("tr")
+    rows = table.find("tbody").find_all("tr")
+
+
+    # Extract coloumn headers
+    headers = ['player_name']
+    headers.extend([col.text.strip() for col in header_row.find_all("th")][2:])
+    headers.extend(['team', 'season'])  # Add 'team' and 'season' fields to headers
+
+    # Extract rows as dictionaries
+    data = []
+    for row in rows:
+        cols = row.find_all("td")
+        cols = [col.text.strip() for col in cols]
+        row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)}
+        row_dict['team'] = team
+        row_dict['season'] = season
+        data.append(row_dict)
+
+
+    # locate current directory
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    
+    # save as CSV file
+    csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_player_salary_output.csv')
+    with open(csv_output_path, "w", newline="") as csvfile:
+        writer = csv.DictWriter(csvfile, fieldnames=headers)
+        writer.writeheader()
+        for row in data:
+            writer.writerow(row)
+
+    # Save as JSON
+    json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_player_salary_output.json')
+    with open(json_output_path, 'w') as jsonfile:
+        json.dump(data, jsonfile)
+
+    # save data to database
+    
+    # for row in data:
+        
+    #     row['Salary'] = int(row['Salary'].replace(',', '').replace('$', '')) if row['Salary'] else None
+        
+    #     player_data = PlayerSalaryData(
+    #         player_name = row['player_name'],
+    #         salary = row['Salary'],
+    #         team = row['team'],
+    #         season = row['season'],       
+    #     )
+    #     player_data.save()
+
+print('success')
+        
+
+