mirror of
https://github.com/nprasad2077/nbaStats.git
synced 2026-11-04 09:15:35 +00:00
added and tested scraper for totals and advanced data
This commit is contained in:
nprasad2077
@@ -0,0 +1,80 @@
|
||||
# Generated by Django 4.2 on 2023-04-27 18:26
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('nba_data', '0005_alter_playerdata_ast_alter_playerdata_blk_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='PlayerAdvancedData',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('player_name', models.CharField(max_length=255, null=True)),
|
||||
('age', models.IntegerField(null=True)),
|
||||
('games', models.IntegerField(null=True)),
|
||||
('minutes_played', models.IntegerField(null=True)),
|
||||
('PER', models.DecimalField(decimal_places=1, max_digits=5, null=True)),
|
||||
('TS_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)),
|
||||
('three_p_attempt_rate', models.DecimalField(decimal_places=3, max_digits=5, null=True)),
|
||||
('ft_attempt_rate', models.DecimalField(decimal_places=3, max_digits=5, null=True)),
|
||||
('orb_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('drb_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('trb_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('ast_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('stl_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('blk_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('tov_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('usg_percent', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('ows', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('dws', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('ws', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('ws_per_48', models.DecimalField(decimal_places=3, max_digits=5, null=True)),
|
||||
('obpm', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('dbpm', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('bpm', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('vorp', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('team', models.CharField(default='', max_length=30)),
|
||||
('season', models.IntegerField(null=True)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='PlayerTotalsData',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('player_name', models.CharField(max_length=255, null=True)),
|
||||
('age', models.IntegerField(null=True)),
|
||||
('games', models.IntegerField(null=True)),
|
||||
('games_started', models.IntegerField(null=True)),
|
||||
('minutes_played', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('field_goals', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('field_attempts', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('field_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)),
|
||||
('three_fg', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('three_attempts', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('three_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)),
|
||||
('two_fg', models.DecimalField(decimal_places=3, max_digits=5, null=True)),
|
||||
('two_attempts', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('two_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)),
|
||||
('effect_fg_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)),
|
||||
('ft', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('fta', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('ft_percent', models.DecimalField(decimal_places=3, max_digits=5, null=True)),
|
||||
('ORB', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('DRB', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('TRB', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('AST', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('STL', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('BLK', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('TOV', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('PF', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('PTS', models.DecimalField(decimal_places=2, max_digits=5, null=True)),
|
||||
('team', models.CharField(default='', max_length=30)),
|
||||
('season', models.IntegerField(null=True)),
|
||||
],
|
||||
),
|
||||
]
|
||||
@@ -95,6 +95,8 @@ class PlayerAdvancedData(models.Model):
|
||||
dbpm = models.DecimalField(max_digits=5, decimal_places=2, null=True)
|
||||
bpm = models.DecimalField(max_digits=5, decimal_places=2, null=True)
|
||||
vorp = models.DecimalField(max_digits=5, decimal_places=2, null=True)
|
||||
team = models.CharField(max_length=30, default='')
|
||||
season = models.IntegerField(null=True)
|
||||
|
||||
def __str__(self):
|
||||
return self.player_name
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import os
|
||||
import sys
|
||||
import django
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import csv
|
||||
import json
|
||||
import time
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings')
|
||||
django.setup()
|
||||
|
||||
from nba_data.models import PlayerAdvancedData
|
||||
|
||||
teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHO']
|
||||
team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS']
|
||||
|
||||
# team = input('What team? ')
|
||||
# OMIT above. Will now only ask for season and iterate thru teams list.
|
||||
season = input('What season? ')
|
||||
|
||||
for team in teams:
|
||||
url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html'
|
||||
response = requests.get(url_make)
|
||||
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
|
||||
table = soup.find("table", {"id": "advanced"})
|
||||
|
||||
if table is None:
|
||||
print(f"Table not found for team {team} and season {season}.")
|
||||
# sys.exit(1)
|
||||
continue
|
||||
|
||||
header_row = table.find("thead").find("tr")
|
||||
rows = table.find("tbody").find_all("tr")
|
||||
|
||||
|
||||
# Extract coloumn headers
|
||||
headers = [col.text.strip() for col in header_row.find_all("th")][1:]
|
||||
headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers
|
||||
|
||||
# Extract rows as dictionaries
|
||||
data = []
|
||||
for row in rows:
|
||||
cols = row.find_all("td")
|
||||
cols = [col.text.strip() for col in cols]
|
||||
row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)}
|
||||
row_dict['team'] = team
|
||||
row_dict['season'] = season
|
||||
data.append(row_dict)
|
||||
|
||||
|
||||
# locate current directory
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# save as CSV file
|
||||
csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_advanced_output.csv')
|
||||
with open(csv_output_path, "w", newline="") as csvfile:
|
||||
writer = csv.DictWriter(csvfile, fieldnames=headers)
|
||||
writer.writeheader()
|
||||
for row in data:
|
||||
writer.writerow(row)
|
||||
|
||||
# Save as JSON
|
||||
json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_advanced_output.json')
|
||||
with open(json_output_path, 'w') as jsonfile:
|
||||
json.dump(data, jsonfile)
|
||||
|
||||
# # save data to database
|
||||
# for row in data:
|
||||
# player_data = PlayerData(
|
||||
# name=row['Player'],
|
||||
# age=row['Age'],
|
||||
# games=row['G'],
|
||||
# games_started=row['GS'],
|
||||
# minutes_pg=row['MP'],
|
||||
# field_goals=row['FG'],
|
||||
# field_attempts=row['FGA'],
|
||||
# field_percent=row['FG%'],
|
||||
# three_fg=row['3P'],
|
||||
# three_attempts=row['3PA'],
|
||||
# three_percent=row['3P%'],
|
||||
# two_fg=row['2P'],
|
||||
# two_attempts=row['2PA'],
|
||||
# two_percent=row['2P%'],
|
||||
# effect_fg_percent=row['eFG%'],
|
||||
# ft=row['FT'],
|
||||
# fta=row['FTA'],
|
||||
# ft_percent=row['FT%'],
|
||||
# ORB=row['ORB'],
|
||||
# DRB=row['DRB'],
|
||||
# TRB=row['TRB'],
|
||||
# AST=row['AST'],
|
||||
# STL=row['STL'],
|
||||
# BLK=row['BLK'],
|
||||
# TOV=row['TOV'],
|
||||
# PF=row['PF'],
|
||||
# PTS=row['PTS'],
|
||||
# team=row['team'],
|
||||
# season=row['season'],
|
||||
# )
|
||||
# player_data.save()
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import os
|
||||
import sys
|
||||
import django
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import csv
|
||||
import json
|
||||
import time
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings')
|
||||
django.setup()
|
||||
|
||||
from nba_data.models import PlayerTotalsData
|
||||
|
||||
teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHO']
|
||||
team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS']
|
||||
|
||||
# team = input('What team? ')
|
||||
# OMIT above. Will now only ask for season and iterate thru teams list.
|
||||
season = input('What season? ')
|
||||
|
||||
for team in teams:
|
||||
url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html'
|
||||
response = requests.get(url_make)
|
||||
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
|
||||
table = soup.find("table", {"id": "totals"})
|
||||
|
||||
if table is None:
|
||||
print(f"Table not found for team {team} and season {season}.")
|
||||
# sys.exit(1)
|
||||
continue
|
||||
|
||||
header_row = table.find("thead").find("tr")
|
||||
rows = table.find("tbody").find_all("tr")
|
||||
|
||||
|
||||
# Extract coloumn headers
|
||||
headers = [col.text.strip() for col in header_row.find_all("th")][1:]
|
||||
headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers
|
||||
|
||||
# Extract rows as dictionaries
|
||||
data = []
|
||||
for row in rows:
|
||||
cols = row.find_all("td")
|
||||
cols = [col.text.strip() for col in cols]
|
||||
row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)}
|
||||
row_dict['team'] = team
|
||||
row_dict['season'] = season
|
||||
data.append(row_dict)
|
||||
|
||||
|
||||
# locate current directory
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# save as CSV file
|
||||
csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_totals_output.csv')
|
||||
with open(csv_output_path, "w", newline="") as csvfile:
|
||||
writer = csv.DictWriter(csvfile, fieldnames=headers)
|
||||
writer.writeheader()
|
||||
for row in data:
|
||||
writer.writerow(row)
|
||||
|
||||
# Save as JSON
|
||||
json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_totals_output.json')
|
||||
with open(json_output_path, 'w') as jsonfile:
|
||||
json.dump(data, jsonfile)
|
||||
|
||||
# # save data to database
|
||||
# for row in data:
|
||||
# player_data = PlayerData(
|
||||
# name=row['Player'],
|
||||
# age=row['Age'],
|
||||
# games=row['G'],
|
||||
# games_started=row['GS'],
|
||||
# minutes_pg=row['MP'],
|
||||
# field_goals=row['FG'],
|
||||
# field_attempts=row['FGA'],
|
||||
# field_percent=row['FG%'],
|
||||
# three_fg=row['3P'],
|
||||
# three_attempts=row['3PA'],
|
||||
# three_percent=row['3P%'],
|
||||
# two_fg=row['2P'],
|
||||
# two_attempts=row['2PA'],
|
||||
# two_percent=row['2P%'],
|
||||
# effect_fg_percent=row['eFG%'],
|
||||
# ft=row['FT'],
|
||||
# fta=row['FTA'],
|
||||
# ft_percent=row['FT%'],
|
||||
# ORB=row['ORB'],
|
||||
# DRB=row['DRB'],
|
||||
# TRB=row['TRB'],
|
||||
# AST=row['AST'],
|
||||
# STL=row['STL'],
|
||||
# BLK=row['BLK'],
|
||||
# TOV=row['TOV'],
|
||||
# PF=row['PF'],
|
||||
# PTS=row['PTS'],
|
||||
# team=row['team'],
|
||||
# season=row['season'],
|
||||
# )
|
||||
# player_data.save()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user