diff --git a/db.sqlite3 b/db.sqlite3 new file mode 100644 index 0000000..53c32b8 Binary files /dev/null and b/db.sqlite3 differ diff --git a/db.sqlite3-journal b/db.sqlite3-journal new file mode 100644 index 0000000..b6c4cc6 Binary files /dev/null and b/db.sqlite3-journal differ diff --git a/nba_data/admin.py b/nba_data/admin.py index 8c38f3f..1e06526 100644 --- a/nba_data/admin.py +++ b/nba_data/admin.py @@ -1,3 +1,6 @@ from django.contrib import admin +from .models import PlayerData # Register your models here. + +admin.site.register(PlayerData) diff --git a/nba_data/migrations/0001_initial.py b/nba_data/migrations/0001_initial.py new file mode 100644 index 0000000..dff305d --- /dev/null +++ b/nba_data/migrations/0001_initial.py @@ -0,0 +1,47 @@ +# Generated by Django 4.2 on 2023-04-14 20:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='PlayerData', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ('age', models.CharField(max_length=100)), + ('games', models.CharField(max_length=100)), + ('games_started', models.CharField(max_length=100)), + ('minutes_pg', models.CharField(max_length=100)), + ('field_goals', models.CharField(max_length=100)), + ('field_attempts', models.CharField(max_length=100)), + ('field_percent', models.CharField(max_length=100)), + ('three_fg', models.CharField(max_length=100)), + ('three_attempts', models.CharField(max_length=100)), + ('three_percent', models.CharField(max_length=100)), + ('two_fg', models.CharField(max_length=100)), + ('two_attempts', models.CharField(max_length=100)), + ('two_percent', models.CharField(max_length=100)), + ('effect_fg_percent', models.CharField(max_length=100)), + ('ft', models.CharField(max_length=100)), + ('fta', models.CharField(max_length=100)), + ('ft_percent', models.CharField(max_length=100)), + ('ORB', models.CharField(max_length=100)), + ('DRB', models.CharField(max_length=100)), + ('TRB', models.CharField(max_length=100)), + ('AST', models.CharField(max_length=100)), + ('STL', models.CharField(max_length=100)), + ('BLK', models.CharField(max_length=100)), + ('TOV', models.CharField(max_length=100)), + ('PF', models.CharField(max_length=100)), + ('PTS', models.CharField(max_length=100)), + ], + ), + ] diff --git a/nba_data/migrations/0002_alter_playerdata_ast_alter_playerdata_blk_and_more.py b/nba_data/migrations/0002_alter_playerdata_ast_alter_playerdata_blk_and_more.py new file mode 100644 index 0000000..c953f7a --- /dev/null +++ b/nba_data/migrations/0002_alter_playerdata_ast_alter_playerdata_blk_and_more.py @@ -0,0 +1,148 @@ +# Generated by Django 4.2 on 2023-04-18 19:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nba_data', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='playerdata', + name='AST', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='BLK', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='DRB', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='ORB', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='PF', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='PTS', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='STL', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='TOV', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='TRB', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='age', + field=models.IntegerField(), + ), + migrations.AlterField( + model_name='playerdata', + name='effect_fg_percent', + field=models.DecimalField(decimal_places=3, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='field_attempts', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='field_goals', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='field_percent', + field=models.DecimalField(decimal_places=3, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='ft', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='ft_percent', + field=models.DecimalField(decimal_places=3, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='fta', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='games', + field=models.IntegerField(), + ), + migrations.AlterField( + model_name='playerdata', + name='games_started', + field=models.IntegerField(), + ), + migrations.AlterField( + model_name='playerdata', + name='minutes_pg', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='name', + field=models.CharField(max_length=255), + ), + migrations.AlterField( + model_name='playerdata', + name='three_attempts', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='three_fg', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='three_percent', + field=models.DecimalField(decimal_places=3, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='two_attempts', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='two_fg', + field=models.DecimalField(decimal_places=2, max_digits=5), + ), + migrations.AlterField( + model_name='playerdata', + name='two_percent', + field=models.DecimalField(decimal_places=3, max_digits=5), + ), + ] diff --git a/nba_data/migrations/0003_playerdata_season_playerdata_team.py b/nba_data/migrations/0003_playerdata_season_playerdata_team.py new file mode 100644 index 0000000..0a3f191 --- /dev/null +++ b/nba_data/migrations/0003_playerdata_season_playerdata_team.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2 on 2023-04-18 19:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nba_data', '0002_alter_playerdata_ast_alter_playerdata_blk_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='playerdata', + name='season', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='playerdata', + name='team', + field=models.CharField(default='', max_length=30), + ), + ] diff --git a/nba_data/migrations/0004_alter_playerdata_season.py b/nba_data/migrations/0004_alter_playerdata_season.py new file mode 100644 index 0000000..2101cd0 --- /dev/null +++ b/nba_data/migrations/0004_alter_playerdata_season.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2 on 2023-04-18 19:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nba_data', '0003_playerdata_season_playerdata_team'), + ] + + operations = [ + migrations.AlterField( + model_name='playerdata', + name='season', + field=models.IntegerField(null=True), + ), + ] diff --git a/nba_data/migrations/0005_alter_playerdata_ast_alter_playerdata_blk_and_more.py b/nba_data/migrations/0005_alter_playerdata_ast_alter_playerdata_blk_and_more.py new file mode 100644 index 0000000..34bf232 --- /dev/null +++ b/nba_data/migrations/0005_alter_playerdata_ast_alter_playerdata_blk_and_more.py @@ -0,0 +1,148 @@ +# Generated by Django 4.2 on 2023-04-18 19:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nba_data', '0004_alter_playerdata_season'), + ] + + operations = [ + migrations.AlterField( + model_name='playerdata', + name='AST', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='BLK', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='DRB', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='ORB', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='PF', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='PTS', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='STL', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='TOV', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='TRB', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='age', + field=models.IntegerField(null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='effect_fg_percent', + field=models.DecimalField(decimal_places=3, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='field_attempts', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='field_goals', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='field_percent', + field=models.DecimalField(decimal_places=3, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='ft', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='ft_percent', + field=models.DecimalField(decimal_places=3, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='fta', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='games', + field=models.IntegerField(null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='games_started', + field=models.IntegerField(null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='minutes_pg', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='name', + field=models.CharField(max_length=255, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='three_attempts', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='three_fg', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='three_percent', + field=models.DecimalField(decimal_places=3, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='two_attempts', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='two_fg', + field=models.DecimalField(decimal_places=2, max_digits=5, null=True), + ), + migrations.AlterField( + model_name='playerdata', + name='two_percent', + field=models.DecimalField(decimal_places=3, max_digits=5, null=True), + ), + ] diff --git a/nba_data/models.py b/nba_data/models.py index 71a8362..d555929 100644 --- a/nba_data/models.py +++ b/nba_data/models.py @@ -1,3 +1,37 @@ from django.db import models # Create your models here. +class PlayerData(models.Model): + name = models.CharField(max_length=255, null=True) + age = models.IntegerField(null=True) + games = models.IntegerField(null=True) + games_started = models.IntegerField(null=True) + minutes_pg = models.DecimalField(max_digits=5, decimal_places=2, null=True) + field_goals = models.DecimalField(max_digits=5, decimal_places=2, null=True) + field_attempts = models.DecimalField(max_digits=5, decimal_places=2, null=True) + field_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) + three_fg = models.DecimalField(max_digits=5, decimal_places=2, null=True) + three_attempts = models.DecimalField(max_digits=5, decimal_places=2, null=True) + three_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) + two_fg = models.DecimalField(max_digits=5, decimal_places=2, null=True) + two_attempts = models.DecimalField(max_digits=5, decimal_places=2, null=True) + two_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) + effect_fg_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) + ft = models.DecimalField(max_digits=5, decimal_places=2, null=True) + fta = models.DecimalField(max_digits=5, decimal_places=2, null=True) + ft_percent = models.DecimalField(max_digits=5, decimal_places=3, null=True) + ORB= models.DecimalField(max_digits=5, decimal_places=2, null=True) + DRB = models.DecimalField(max_digits=5, decimal_places=2, null=True) + TRB = models.DecimalField(max_digits=5, decimal_places=2, null=True) + AST = models.DecimalField(max_digits=5, decimal_places=2, null=True) + STL = models.DecimalField(max_digits=5, decimal_places=2, null=True) + BLK = models.DecimalField(max_digits=5, decimal_places=2, null=True) + TOV = models.DecimalField(max_digits=5, decimal_places=2, null=True) + PF = models.DecimalField(max_digits=5, decimal_places=2, null=True) + PTS = models.DecimalField(max_digits=5, decimal_places=2, null=True) + team = models.CharField(max_length=30, default='') + season = models.IntegerField(null=True) + + def __str__(self): + return self.name + diff --git a/nba_data/scraper.py b/nba_data/scraper.py index f34d1f5..b329f4d 100644 --- a/nba_data/scraper.py +++ b/nba_data/scraper.py @@ -1,13 +1,28 @@ +import os +import sys +import django import requests from bs4 import BeautifulSoup import csv import json -import os + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nba_stats.settings') +django.setup() + +from nba_data.models import PlayerData + +team = input('What team? ') +season = input('What season? ') + +url_make = 'https://www.basketball-reference.com/teams/' + team + '/' + season + '.html' url = 'https://www.basketball-reference.com/teams/HOU/2018.html' lakers = 'https://www.basketball-reference.com/teams/LAL/2020.html' -response = requests.get(url) +response = requests.get(url_make) soup = BeautifulSoup(response.text, "html.parser") @@ -18,23 +33,24 @@ rows = table.find("tbody").find_all("tr") # Extract coloumn headers headers = [col.text.strip() for col in header_row.find_all("th")][1:] +headers.extend(['team', 'season']) # Add 'team' and 'season' fields to headers - -# extract rows as dictionaries +# Extract rows as dictionaries data = [] for row in rows: cols = row.find_all("td") cols = [col.text.strip() for col in cols] - row_dict = dict(zip(headers, cols)) + row_dict = {header: (None if col == "" else col) for header, col in zip(headers, cols)} + row_dict['team'] = team + row_dict['season'] = season data.append(row_dict) - -## locate current directory + + +# locate current directory script_dir = os.path.dirname(os.path.abspath(__file__)) -## CSV file - +# CSV file csv_output_path = os.path.join(script_dir, '..', 'data', 'output.csv') - with open(csv_output_path, "w", newline="") as csvfile: writer = csv.DictWriter(csvfile, fieldnames =headers) writer.writeheader() @@ -42,7 +58,41 @@ with open(csv_output_path, "w", newline="") as csvfile: writer.writerow(row) # Save as JSON - json_output_path = os.path.join(script_dir, '..', 'data', 'output.json') with open(json_output_path, 'w') as jsonfile: - json.dump(data, jsonfile) \ No newline at end of file + json.dump(data, jsonfile) + +# save data to database +for row in data: + player_data = PlayerData( + name=row['Player'], + age=row['Age'], + games=row['G'], + games_started=row['GS'], + minutes_pg=row['MP'], + field_goals=row['FG'], + field_attempts=row['FGA'], + field_percent=row['FG%'], + three_fg=row['3P'], + three_attempts=row['3PA'], + three_percent=row['3P%'], + two_fg=row['2P'], + two_attempts=row['2PA'], + two_percent=row['2P%'], + effect_fg_percent=row['eFG%'], + ft=row['FT'], + fta=row['FTA'], + ft_percent=row['FT%'], + ORB=row['ORB'], + DRB=row['DRB'], + TRB=row['TRB'], + AST=row['AST'], + STL=row['STL'], + BLK=row['BLK'], + TOV=row['TOV'], + PF=row['PF'], + PTS=row['PTS'], + team=row['team'], + season=row['season'], + ) + player_data.save() diff --git a/nba_data/urls.py b/nba_data/urls.py new file mode 100644 index 0000000..100434e --- /dev/null +++ b/nba_data/urls.py @@ -0,0 +1,6 @@ +from django.urls import path +from .views import get_player_data + +urlpatterns = [ + path('api/player_data/', get_player_data, name='player_data') +] \ No newline at end of file diff --git a/nba_data/views.py b/nba_data/views.py index 91ea44a..10f8362 100644 --- a/nba_data/views.py +++ b/nba_data/views.py @@ -1,3 +1,10 @@ from django.shortcuts import render +from django.http import JsonResponse +from django.core import serializers +from .models import PlayerData # Create your views here. + +def get_player_data(request): + data = serializers.serialize('json', PlayerData.objects.all()) + return JsonResponse(data, safe=False) \ No newline at end of file diff --git a/nba_stats/settings.py b/nba_stats/settings.py index 6c29440..128942d 100644 --- a/nba_stats/settings.py +++ b/nba_stats/settings.py @@ -33,6 +33,7 @@ ALLOWED_HOSTS = [] INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', + 'nba_data', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', diff --git a/nba_stats/urls.py b/nba_stats/urls.py index 505bd88..970cf52 100644 --- a/nba_stats/urls.py +++ b/nba_stats/urls.py @@ -15,8 +15,9 @@ Including another URLconf 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin -from django.urls import path +from django.urls import path, include urlpatterns = [ + path('', include('nba_data.urls')), path('admin/', admin.site.urls), ]