scraped until 2016

nba_data/nba_scraper_shot_chart.py

@@ -13,15 +13,17 @@ django.setup()
 from nba_data.models import PlayerShotChartData
 
 player_name = 'LeBron James'
-season = 2022
+season = 2016
 # team = 'LAL'  # You might want to find a way to get this from the webpage
-url = 'https://www.basketball-reference.com/players/j/jamesle01/shooting/' + str(season)
+url = 'https://www.basketball-reference.com/players/j/jamesle01/shooting/' + \
+    str(season)
 
 response = requests.get(url)
 soup = BeautifulSoup(response.text, "html.parser")
 
 # Find the comment containing the shot chart
-comment = soup.find(string=lambda text: isinstance(text, Comment) and "div_shot-chart" in text)
+comment = soup.find(string=lambda text: isinstance(
+    text, Comment) and "div_shot-chart" in text)
 shot_chart_soup = BeautifulSoup(comment, "html.parser")
 
 # Now you can find the shot chart in the parsed comment
@@ -33,60 +35,67 @@ shots = shot_chart.find_all("div", {"class": ["tooltip make", "tooltip miss"]})
 shot_chart_data = []
 
 for shot_div in shots:
-    # Parse shot data
+    try:
-    style_values = shot_div['style'].split(';')
+        # Parse shot data
-    top_value = int(style_values[0].split(':')[1].replace('px', '').strip())
+        style_values = shot_div['style'].split(';')
-    left_value = int(style_values[1].split(':')[1].replace('px', '').strip())
+        top_value = int(style_values[0].split(
+            ':')[1].replace('px', '').strip())
+        left_value = int(style_values[1].split(
+            ':')[1].replace('px', '').strip())
 
-    # Parse tip attribute
+        # Parse tip attribute
-    tip_parts = shot_div['tip'].split('<br>')
+        tip_parts = shot_div['tip'].split('<br>')
 
-    # Within your loop:
+        # Within your loop:
-    date_and_opponent = re.split(' at | vs ', tip_parts[0])
+        date_and_opponent = re.split(' at | vs ', tip_parts[0])
-    date = date_and_opponent[0].strip().split(',')[0] + ',' + date_and_opponent[0].strip().split(',')[1]
+        date = date_and_opponent[0].strip().split(
-    team = date_and_opponent[0].strip().split(',')[2].strip()
+            ',')[0] + ',' + date_and_opponent[0].strip().split(',')[1]
-    opponent = date_and_opponent[1].strip()
+        team = date_and_opponent[0].strip().split(',')[2].strip()
+        opponent = date_and_opponent[1].strip()
 
-    quarter_and_time = tip_parts[1].split(',')
+        quarter_and_time = tip_parts[1].split(',')
-    quarter = quarter_and_time[0].strip()
+        quarter = quarter_and_time[0].strip()
-    time_remaining = quarter_and_time[1].strip().split(' ')[0]
+        time_remaining = quarter_and_time[1].strip().split(' ')[0]
 
-    result_and_type_and_distance = tip_parts[2].split(' ')
+        result_and_type_and_distance = tip_parts[2].split(' ')
-    result = result_and_type_and_distance[0].strip() == 'Made'
+        result = result_and_type_and_distance[0].strip() == 'Made'
-    shot_type = int(result_and_type_and_distance[1][0])
+        shot_type = int(result_and_type_and_distance[1][0])
-    distance_ft = int(result_and_type_and_distance[-2].strip())
+        distance_ft = int(result_and_type_and_distance[-2].strip())
 
-    team_score_and_opponent_score = tip_parts[3].split(' ')[-1].split('-')
+        team_score_and_opponent_score = tip_parts[3].split(' ')[-1].split('-')
-    team_score = int(team_score_and_opponent_score[0].strip())
+        team_score = int(team_score_and_opponent_score[0].strip())
-    opponent_score = int(team_score_and_opponent_score[1].strip())
+        opponent_score = int(team_score_and_opponent_score[1].strip())
 
-    lead = team_score > opponent_score
+        lead = team_score > opponent_score
 
-    shot_chart_row = {
+        shot_chart_row = {
-        'player_name': player_name,
+            'player_name': player_name,
-        'top': top_value,
+            'top': top_value,
-        'left': left_value,
+            'left': left_value,
-        'date': date,
+            'date': date,
-        'qtr': quarter,
+            'qtr': quarter,
-        'time_remaining': time_remaining,
+            'time_remaining': time_remaining,
-        'result': result,
+            'result': result,
-        'shot_type': shot_type,
+            'shot_type': shot_type,
-        'distance_ft': distance_ft,
+            'distance_ft': distance_ft,
-        'lead': lead,
+            'lead': lead,
-        'lebron_team_score': team_score,
+            'lebron_team_score': team_score,
-        'opponent_team_score': opponent_score,
+            'opponent_team_score': opponent_score,
-        'opponent': opponent,
+            'opponent': opponent,
-        'team': team,
+            'team': team,
-        'season': season
+            'season': season
-    }
+        }
-    shot_chart_data.append(shot_chart_row)
+        shot_chart_data.append(shot_chart_row)
+
+    except Exception as e:
+        print(f"Error processing shot_div: {shot_div}")
+        print(f"Exception: {e}")
 
-    # shot_chart_entry = PlayerShotChartData(**shot_chart_row)
-    # shot_chart_entry.save()
 
 # save as CSV file
 script_dir = os.path.dirname(os.path.abspath(__file__))
-csv_output_path = os.path.join(script_dir, '..', 'data', f'{player_name}_{season}_shot_chart_output.csv')
+csv_output_path = os.path.join(
+    script_dir, '..', 'data', f'{player_name}_{season}_shot_chart_output.csv')
 with open(csv_output_path, "w", newline="") as csvfile:
     writer = csv.DictWriter(csvfile, fieldnames=shot_chart_data[0].keys())
     writer.writeheader()
@@ -94,9 +103,11 @@ with open(csv_output_path, "w", newline="") as csvfile:
         writer.writerow(row)
 
 # Save the records in the database
-PlayerShotChartData.objects.bulk_create(
+try:
-    PlayerShotChartData(**row) for row in shot_chart_data
+    PlayerShotChartData.objects.bulk_create(
-)
+        PlayerShotChartData(**row) for row in shot_chart_data
-
+    )
+except Exception as e:
+    print(f"Error inserting data into database: {e}")
 
 print('success')