nprasad2077/nbaStats
1
0
Fork 0
mirror of https://github.com/nprasad2077/nbaStats.git synced 2026-11-04 09:15:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

advanced and totals data scraped success

Browse Source
This commit is contained in:
nprasad2077
2023-04-27 17:57:26 -05:00
parent 9a845ade4a
commit daf96cd270
2 changed files with 141 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nba_data/nba_scraper_advanced.py
+34 -36
View File
@@ -15,11 +15,10 @@ django.setup()
from nba_data.models import PlayerAdvancedData
# If season is 2014, then change CHO to CHA
teams = ['HOU' ,'PHI', 'BOS', 'NYK', 'BRK', 'TOR', 'MEM', 'NOP', 'DAL', 'SAS', 'DEN', 'MIN', 'OKC', 'UTA', 'POR', 'MIL', 'CLE', 'CHI', 'IND', 'DET', 'SAC', 'PHO', 'GSW', 'LAC', 'LAL', 'MIA', 'ATL', 'WAS', 'ORL', 'CHO']
team_abbreviations = ['ATL', 'BOS', 'BRK', 'CHI', 'CHO', 'CLE', 'DAL', 'DEN', 'DET', 'GSW', 'HOU', 'IND', 'LAC', 'LAL', 'MEM', 'MIA', 'MIL', 'MIN', 'NOP', 'NYK', 'OKC', 'ORL', 'PHI', 'PHO', 'POR', 'SAC', 'SAS', 'TOR', 'UTA', 'WAS']
# team = input('What team? ')
# OMIT above. Will now only ask for season and iterate thru teams list.
season = input('What season? ')
for team in teams:
@@ -70,40 +69,39 @@ for team in teams:
with open(json_output_path, 'w') as jsonfile:
json.dump(data, jsonfile)
# # save data to database
# for row in data:
# player_data = PlayerData(
# name=row['Player'],
# age=row['Age'],
# games=row['G'],
# games_started=row['GS'],
# minutes_pg=row['MP'],
# field_goals=row['FG'],
# field_attempts=row['FGA'],
# field_percent=row['FG%'],
# three_fg=row['3P'],
# three_attempts=row['3PA'],
# three_percent=row['3P%'],
# two_fg=row['2P'],
# two_attempts=row['2PA'],
# two_percent=row['2P%'],
# effect_fg_percent=row['eFG%'],
# ft=row['FT'],
# fta=row['FTA'],
# ft_percent=row['FT%'],
# ORB=row['ORB'],
# DRB=row['DRB'],
# TRB=row['TRB'],
# AST=row['AST'],
# STL=row['STL'],
# BLK=row['BLK'],
# TOV=row['TOV'],
# PF=row['PF'],
# PTS=row['PTS'],
# team=row['team'],
# season=row['season'],
# )
# player_data.save()
# save data to database
for row in data:
player_data = PlayerAdvancedData(
player_name = row['Player'],
age = row['Age'],
games = row['G'],
minutes_played = row['MP'],
PER = row['PER'],
TS_percent = row['TS%'],
three_p_attempt_rate = row['3PAr'],
ft_attempt_rate = row['FTr'],
orb_percent = row['ORB%'],
drb_percent = row['DRB%'],
trb_percent = row['TRB%'],
ast_percent = row['AST%'],
stl_percent = row['STL%'],
blk_percent = row['BLK%'],
tov_percent = row['TOV%'],
usg_percent = row['USG%'],
ows = row['OWS'],
dws = row['DWS'],
ws = row['WS'],
ws_per_48 = row['WS/48'],
obpm = row['OBPM'],
dbpm = row['DBPM'],
bpm = row['BPM'],
vorp = row['VORP'],
team = row['team'],
season = row['season'],
)
player_data.save()
print('success')
nba_data/nba_scraper_totals_single.py
+107
View File
@@ -0,0 +1,107 @@
import os
import sys
import django
import requests
from bs4 import BeautifulSoup
import csv
import json
import time
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings')
django.setup()
from nba_data.models import PlayerTotalsData
season = input('What season? ')
team = input('What team?')
url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html'
response = requests.get(url_make)
soup = BeautifulSoup(response.text, "html.parser")
table = soup.find("table", {"id": "totals"})
if table is None:
print(f"Table not found for team {team} and season {season}.")
# sys.exit(1)
# continue
header_row = table.find("thead").find("tr")
rows = table.find("tbody").find_all("tr")
# Extract coloumn headers
headers = [col.text.strip() for col in header_row.find_all("th")][1:]
headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers
# Extract rows as dictionaries
data = []
for row in rows:
cols = row.find_all("td")
cols = [col.text.strip() for col in cols]
row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)}
row_dict['team'] = team
row_dict['season'] = season
data.append(row_dict)
# locate current directory
script_dir = os.path.dirname(os.path.abspath(__file__))
# save as CSV file
csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_totals_output.csv')
with open(csv_output_path, "w", newline="") as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=headers)
writer.writeheader()
for row in data:
writer.writerow(row)
# Save as JSON
json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_totals_output.json')
with open(json_output_path, 'w') as jsonfile:
json.dump(data, jsonfile)
# save data to database
for row in data:
player_data = PlayerTotalsData(
player_name = row['Player'],
age = row['Age'],
games = row['G'],
games_started = row['GS'],
minutes_played = row['MP'],
field_goals = row['FG'],
field_attempts = row['FGA'],
field_percent = row['FG%'],
three_fg = row['3P'],
three_attempts = row['3PA'],
three_percent = row['3P%'],
two_fg = row['2P'],
two_attempts = row['2PA'],
two_percent = row['2P%'],
effect_fg_percent = row['eFG%'],
ft = row['FT'],
fta = row['FTA'],
ft_percent = row['FT%'],
ORB= row['ORB'],
DRB = row['DRB'],
TRB = row['TRB'],
AST = row['AST'],
STL = row['STL'],
BLK = row['BLK'],
TOV = row['TOV'],
PF = row['PF'],
PTS = row['PTS'],
team = row['team'],
season = row['season'],
)
player_data.save()
print('success')