From 1c54ed2627fac1185af1283937a477ca571feece Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Wed, 10 May 2023 01:23:50 -0500 Subject: [PATCH] scraped until 2016 --- nba_data/nba_scraper_shot_chart.py | 115 ++++++++++++++++------------- 1 file changed, 63 insertions(+), 52 deletions(-) diff --git a/nba_data/nba_scraper_shot_chart.py b/nba_data/nba_scraper_shot_chart.py index 2aab8a7..7bad356 100644 --- a/nba_data/nba_scraper_shot_chart.py +++ b/nba_data/nba_scraper_shot_chart.py @@ -13,15 +13,17 @@ django.setup() from nba_data.models import PlayerShotChartData player_name = 'LeBron James' -season = 2022 +season = 2016 # team = 'LAL' # You might want to find a way to get this from the webpage -url = 'https://www.basketball-reference.com/players/j/jamesle01/shooting/' + str(season) +url = 'https://www.basketball-reference.com/players/j/jamesle01/shooting/' + \ + str(season) response = requests.get(url) soup = BeautifulSoup(response.text, "html.parser") # Find the comment containing the shot chart -comment = soup.find(string=lambda text: isinstance(text, Comment) and "div_shot-chart" in text) +comment = soup.find(string=lambda text: isinstance( + text, Comment) and "div_shot-chart" in text) shot_chart_soup = BeautifulSoup(comment, "html.parser") # Now you can find the shot chart in the parsed comment @@ -33,70 +35,79 @@ shots = shot_chart.find_all("div", {"class": ["tooltip make", "tooltip miss"]}) shot_chart_data = [] for shot_div in shots: - # Parse shot data - style_values = shot_div['style'].split(';') - top_value = int(style_values[0].split(':')[1].replace('px', '').strip()) - left_value = int(style_values[1].split(':')[1].replace('px', '').strip()) + try: + # Parse shot data + style_values = shot_div['style'].split(';') + top_value = int(style_values[0].split( + ':')[1].replace('px', '').strip()) + left_value = int(style_values[1].split( + ':')[1].replace('px', '').strip()) - # Parse tip attribute - tip_parts = shot_div['tip'].split('
') + # Parse tip attribute + tip_parts = shot_div['tip'].split('
') - # Within your loop: - date_and_opponent = re.split(' at | vs ', tip_parts[0]) - date = date_and_opponent[0].strip().split(',')[0] + ',' + date_and_opponent[0].strip().split(',')[1] - team = date_and_opponent[0].strip().split(',')[2].strip() - opponent = date_and_opponent[1].strip() + # Within your loop: + date_and_opponent = re.split(' at | vs ', tip_parts[0]) + date = date_and_opponent[0].strip().split( + ',')[0] + ',' + date_and_opponent[0].strip().split(',')[1] + team = date_and_opponent[0].strip().split(',')[2].strip() + opponent = date_and_opponent[1].strip() - quarter_and_time = tip_parts[1].split(',') - quarter = quarter_and_time[0].strip() - time_remaining = quarter_and_time[1].strip().split(' ')[0] + quarter_and_time = tip_parts[1].split(',') + quarter = quarter_and_time[0].strip() + time_remaining = quarter_and_time[1].strip().split(' ')[0] - result_and_type_and_distance = tip_parts[2].split(' ') - result = result_and_type_and_distance[0].strip() == 'Made' - shot_type = int(result_and_type_and_distance[1][0]) - distance_ft = int(result_and_type_and_distance[-2].strip()) + result_and_type_and_distance = tip_parts[2].split(' ') + result = result_and_type_and_distance[0].strip() == 'Made' + shot_type = int(result_and_type_and_distance[1][0]) + distance_ft = int(result_and_type_and_distance[-2].strip()) - team_score_and_opponent_score = tip_parts[3].split(' ')[-1].split('-') - team_score = int(team_score_and_opponent_score[0].strip()) - opponent_score = int(team_score_and_opponent_score[1].strip()) + team_score_and_opponent_score = tip_parts[3].split(' ')[-1].split('-') + team_score = int(team_score_and_opponent_score[0].strip()) + opponent_score = int(team_score_and_opponent_score[1].strip()) - lead = team_score > opponent_score + lead = team_score > opponent_score - shot_chart_row = { - 'player_name': player_name, - 'top': top_value, - 'left': left_value, - 'date': date, - 'qtr': quarter, - 'time_remaining': time_remaining, - 'result': result, - 'shot_type': shot_type, - 'distance_ft': distance_ft, - 'lead': lead, - 'lebron_team_score': team_score, - 'opponent_team_score': opponent_score, - 'opponent': opponent, - 'team': team, - 'season': season - } - shot_chart_data.append(shot_chart_row) + shot_chart_row = { + 'player_name': player_name, + 'top': top_value, + 'left': left_value, + 'date': date, + 'qtr': quarter, + 'time_remaining': time_remaining, + 'result': result, + 'shot_type': shot_type, + 'distance_ft': distance_ft, + 'lead': lead, + 'lebron_team_score': team_score, + 'opponent_team_score': opponent_score, + 'opponent': opponent, + 'team': team, + 'season': season + } + shot_chart_data.append(shot_chart_row) + + except Exception as e: + print(f"Error processing shot_div: {shot_div}") + print(f"Exception: {e}") - # shot_chart_entry = PlayerShotChartData(**shot_chart_row) - # shot_chart_entry.save() # save as CSV file script_dir = os.path.dirname(os.path.abspath(__file__)) -csv_output_path = os.path.join(script_dir, '..', 'data', f'{player_name}_{season}_shot_chart_output.csv') +csv_output_path = os.path.join( + script_dir, '..', 'data', f'{player_name}_{season}_shot_chart_output.csv') with open(csv_output_path, "w", newline="") as csvfile: writer = csv.DictWriter(csvfile, fieldnames=shot_chart_data[0].keys()) writer.writeheader() for row in shot_chart_data: writer.writerow(row) - + # Save the records in the database -PlayerShotChartData.objects.bulk_create( - PlayerShotChartData(**row) for row in shot_chart_data -) +try: + PlayerShotChartData.objects.bulk_create( + PlayerShotChartData(**row) for row in shot_chart_data + ) +except Exception as e: + print(f"Error inserting data into database: {e}") - -print('success') \ No newline at end of file +print('success')