scraped until 2016

This commit is contained in:
nprasad2077 committed 2023-05-10 01:23:50 -05:00
1 parent cad7e179c3
commit 1c54ed2627
1 file changed
+21 -10
+21 -10
View File
@@ -13,15 +13,17 @@ django.setup()
from nba_data.models import PlayerShotChartData from nba_data.models import PlayerShotChartData
player_name = 'LeBron James' player_name = 'LeBron James'
season = 2022 season = 2016
# team = 'LAL' # You might want to find a way to get this from the webpage # team = 'LAL' # You might want to find a way to get this from the webpage
url = 'https://www.basketball-reference.com/players/j/jamesle01/shooting/' + str(season) url = 'https://www.basketball-reference.com/players/j/jamesle01/shooting/' + \
str(season)
response = requests.get(url) response = requests.get(url)
soup = BeautifulSoup(response.text, "html.parser") soup = BeautifulSoup(response.text, "html.parser")
# Find the comment containing the shot chart # Find the comment containing the shot chart
comment = soup.find(string=lambda text: isinstance(text, Comment) and "div_shot-chart" in text) comment = soup.find(string=lambda text: isinstance(
text, Comment) and "div_shot-chart" in text)
shot_chart_soup = BeautifulSoup(comment, "html.parser") shot_chart_soup = BeautifulSoup(comment, "html.parser")
# Now you can find the shot chart in the parsed comment # Now you can find the shot chart in the parsed comment
@@ -33,17 +35,21 @@ shots = shot_chart.find_all("div", {"class": ["tooltip make", "tooltip miss"]})
shot_chart_data = [] shot_chart_data = []
for shot_div in shots: for shot_div in shots:
try:
# Parse shot data # Parse shot data
style_values = shot_div['style'].split(';') style_values = shot_div['style'].split(';')
top_value = int(style_values[0].split(':')[1].replace('px', '').strip()) top_value = int(style_values[0].split(
left_value = int(style_values[1].split(':')[1].replace('px', '').strip()) ':')[1].replace('px', '').strip())
left_value = int(style_values[1].split(
':')[1].replace('px', '').strip())
# Parse tip attribute # Parse tip attribute
tip_parts = shot_div['tip'].split('<br>') tip_parts = shot_div['tip'].split('<br>')
# Within your loop: # Within your loop:
date_and_opponent = re.split(' at | vs ', tip_parts[0]) date_and_opponent = re.split(' at | vs ', tip_parts[0])
date = date_and_opponent[0].strip().split(',')[0] + ',' + date_and_opponent[0].strip().split(',')[1] date = date_and_opponent[0].strip().split(
',')[0] + ',' + date_and_opponent[0].strip().split(',')[1]
team = date_and_opponent[0].strip().split(',')[2].strip() team = date_and_opponent[0].strip().split(',')[2].strip()
opponent = date_and_opponent[1].strip() opponent = date_and_opponent[1].strip()
@@ -81,12 +87,15 @@ for shot_div in shots:
} }
shot_chart_data.append(shot_chart_row) shot_chart_data.append(shot_chart_row)
# shot_chart_entry = PlayerShotChartData(**shot_chart_row) except Exception as e:
# shot_chart_entry.save() print(f"Error processing shot_div: {shot_div}")
print(f"Exception: {e}")
# save as CSV file # save as CSV file
script_dir = os.path.dirname(os.path.abspath(__file__)) script_dir = os.path.dirname(os.path.abspath(__file__))
csv_output_path = os.path.join(script_dir, '..', 'data', f'{player_name}_{season}_shot_chart_output.csv') csv_output_path = os.path.join(
script_dir, '..', 'data', f'{player_name}_{season}_shot_chart_output.csv')
with open(csv_output_path, "w", newline="") as csvfile: with open(csv_output_path, "w", newline="") as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=shot_chart_data[0].keys()) writer = csv.DictWriter(csvfile, fieldnames=shot_chart_data[0].keys())
writer.writeheader() writer.writeheader()
@@ -94,9 +103,11 @@ with open(csv_output_path, "w", newline="") as csvfile:
writer.writerow(row) writer.writerow(row)
# Save the records in the database # Save the records in the database
try:
PlayerShotChartData.objects.bulk_create( PlayerShotChartData.objects.bulk_create(
PlayerShotChartData(**row) for row in shot_chart_data PlayerShotChartData(**row) for row in shot_chart_data
) )
except Exception as e:
print(f"Error inserting data into database: {e}")
print('success') print('success')