From dcab61a7fce58515ef55b4585819cd047e5455d6 Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Mon, 8 May 2023 19:49:11 -0500 Subject: [PATCH] salary scraper success; --- ...salarydata_season_playersalarydata_team.py | 23 ++++ nba_data/models.py | 2 + nba_data/nba_scraper_playoffs_totals.py | 1 + nba_data/nba_scraper_salary.py | 104 ++++++++++++++++++ 4 files changed, 130 insertions(+) create mode 100644 nba_data/migrations/0013_playersalarydata_season_playersalarydata_team.py diff --git a/nba_data/migrations/0013_playersalarydata_season_playersalarydata_team.py b/nba_data/migrations/0013_playersalarydata_season_playersalarydata_team.py new file mode 100644 index 0000000..0f600b2 --- /dev/null +++ b/nba_data/migrations/0013_playersalarydata_season_playersalarydata_team.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2 on 2023-05-09 00:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nba_data', '0012_playersalarydata'), + ] + + operations = [ + migrations.AddField( + model_name='playersalarydata', + name='season', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='playersalarydata', + name='team', + field=models.CharField(default='', max_length=30), + ), + ] diff --git a/nba_data/models.py b/nba_data/models.py index 6f0f639..0bc5fda 100644 --- a/nba_data/models.py +++ b/nba_data/models.py @@ -203,6 +203,8 @@ class PlayerPlayoffAdvancedData(models.Model): class PlayerSalaryData(models.Model): player_name = models.CharField(max_length=255, null=True) salary = models.IntegerField(null=True) + team = models.CharField(max_length=30, default='') + season = models.IntegerField(null=True) def __str__(self): return self.player_name diff --git a/nba_data/nba_scraper_playoffs_totals.py b/nba_data/nba_scraper_playoffs_totals.py index 202fe41..47853ff 100644 --- a/nba_data/nba_scraper_playoffs_totals.py +++ b/nba_data/nba_scraper_playoffs_totals.py @@ -29,6 +29,7 @@ from nba_data.models import PlayerPlayoffTotalsData teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'NJN', 'TOR', 'MEM', 'NOH', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHA'] + team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS'] season = input('What season? ') diff --git a/nba_data/nba_scraper_salary.py b/nba_data/nba_scraper_salary.py index e69de29..031a428 100644 --- a/nba_data/nba_scraper_salary.py +++ b/nba_data/nba_scraper_salary.py @@ -0,0 +1,104 @@ +import os +import sys +import django +import requests +from bs4 import BeautifulSoup, Comment +import csv +import json +import time +import re + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings') +django.setup() + +from nba_data.models import PlayerSalaryData + +# Change CHO to CHA 2014 and prior. +# Change NOP to NOH for 2013 and prior. +# Change OKC to SEA for 2008 and prior. +# Change NOH to NOK for 2007 and 2006 +# Change NOK to NOH for 2005 and prior. +# Change NOH to CHH for 2002 and prior. NOLA did not have a team before 2003. +# Change WAS to WSB for 1997 and prior. +# Change BRK to NJN for 2012 and prior. +# Change MEM to VAN for 2001 and prior. +# Change LAC to SDC for 1984 and prior. + + +teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHO'] +teams_test = ['HOU', 'PHI', 'BOS'] + +season = input('What season? ') + +for team in teams_test: + url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html' + response = requests.get(url_make) + + soup = BeautifulSoup(response.text, "html.parser") + + comment = soup.find(string=lambda text: isinstance(text, Comment) and "salaries2" in text) + table_soup = BeautifulSoup(comment, "html.parser") + table = table_soup.find("table", {'class': 'suppress_all stats_table', 'id': 'salaries2'}) + + if table is None: + print(f"Table not found for team {team} and season {season}.") + # sys.exit(1) + continue + + header_row = table.find("thead").find("tr") + rows = table.find("tbody").find_all("tr") + + + # Extract coloumn headers + headers = ['player_name'] + headers.extend([col.text.strip() for col in header_row.find_all("th")][2:]) + headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers + + # Extract rows as dictionaries + data = [] + for row in rows: + cols = row.find_all("td") + cols = [col.text.strip() for col in cols] + row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)} + row_dict['team'] = team + row_dict['season'] = season + data.append(row_dict) + + + # locate current directory + script_dir = os.path.dirname(os.path.abspath(__file__)) + + # save as CSV file + csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_player_salary_output.csv') + with open(csv_output_path, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=headers) + writer.writeheader() + for row in data: + writer.writerow(row) + + # Save as JSON + json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_player_salary_output.json') + with open(json_output_path, 'w') as jsonfile: + json.dump(data, jsonfile) + + # save data to database + + # for row in data: + + # row['Salary'] = int(row['Salary'].replace(',', '').replace('$', '')) if row['Salary'] else None + + # player_data = PlayerSalaryData( + # player_name = row['player_name'], + # salary = row['Salary'], + # team = row['team'], + # season = row['season'], + # ) + # player_data.save() + +print('success') + + + \ No newline at end of file