From 857faf17d4f338f7b444472f1f5e79af2f89d0b9 Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 20 Apr 2023 16:42:13 -0500 Subject: [PATCH 01/14] added delete player data in db only script --- nba_data/management/commands/delete_player_data.py | 12 ++++++++++++ nba_data/scraper.py | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 nba_data/management/commands/delete_player_data.py diff --git a/nba_data/management/commands/delete_player_data.py b/nba_data/management/commands/delete_player_data.py new file mode 100644 index 0000000..ece6719 --- /dev/null +++ b/nba_data/management/commands/delete_player_data.py @@ -0,0 +1,12 @@ +from django.core.management.base import BaseCommand +from nba_data.models import PlayerData + +class Command(BaseCommand): + # Provides a description for use with the custom django management command (python manage.py help). + help = 'Delete all player data' + + def handle(self, *args, **options): + # Delete all player data + PlayerData.objects.all().delete() + + self.stdout.write(self.style.SUCCESS('Successfully deleted all players in DB')) \ No newline at end of file diff --git a/nba_data/scraper.py b/nba_data/scraper.py index 2d510b6..de4c42a 100644 --- a/nba_data/scraper.py +++ b/nba_data/scraper.py @@ -32,7 +32,8 @@ for team in teams: if table is None: print(f"Table not found for team {team} and season {season}.") - sys.exit(1) + # sys.exit(1) + continue header_row = table.find("thead").find("tr") rows = table.find("tbody").find_all("tr") From 50af7d9fc83e1bef69b0a4cf1a163e430e14ce1e Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 20 Apr 2023 17:44:47 -0500 Subject: [PATCH 02/14] script to reset SQL table at id 1 added --- .../commands/delete_player_data_and_reset_id.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 nba_data/management/commands/delete_player_data_and_reset_id.py diff --git a/nba_data/management/commands/delete_player_data_and_reset_id.py b/nba_data/management/commands/delete_player_data_and_reset_id.py new file mode 100644 index 0000000..10b2bbf --- /dev/null +++ b/nba_data/management/commands/delete_player_data_and_reset_id.py @@ -0,0 +1,17 @@ +from django.core.management.base import BaseCommand +from nba_data.models import PlayerData +from django.db import connection + +class Command(BaseCommand): + # Provides a description for use with the custom django management command (python manage.py help). + help = 'Delete all player data and reset primary key sequence in DB' + + def handle(self, *args, **options): + # Delete all player data + PlayerData.objects.all().delete() + + # Reset the PK sequence + with connection.cursor() as cursor: + cursor.execute('ALTER SEQUENCE nba_data_playerdata_id_seq RESTART WITH 1;') + + self.stdout.write(self.style.SUCCESS('Successfully deleted all players in DB')) \ No newline at end of file From a08f9b6117fc4fcccc6aa35f47f1b9601cbadf58 Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 20 Apr 2023 18:13:52 -0500 Subject: [PATCH 03/14] change scraper to output new json and csv for each team --- nba_data/scraper.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nba_data/scraper.py b/nba_data/scraper.py index de4c42a..34bc4dd 100644 --- a/nba_data/scraper.py +++ b/nba_data/scraper.py @@ -56,17 +56,17 @@ for team in teams: # locate current directory script_dir = os.path.dirname(os.path.abspath(__file__)) - + # CSV file - csv_output_path = os.path.join(script_dir, '..', 'data', 'output.csv') + csv_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_output.csv') with open(csv_output_path, "w", newline="") as csvfile: - writer = csv.DictWriter(csvfile, fieldnames =headers) + writer = csv.DictWriter(csvfile, fieldnames=headers) writer.writeheader() for row in data: writer.writerow(row) # Save as JSON - json_output_path = os.path.join(script_dir, '..', 'data', 'output.json') + json_output_path = os.path.join(script_dir, '..', 'data', f'{team}_{season}_output.json') with open(json_output_path, 'w') as jsonfile: json.dump(data, jsonfile) @@ -103,4 +103,7 @@ for team in teams: team=row['team'], season=row['season'], ) - player_data.save() \ No newline at end of file + player_data.save() + + + \ No newline at end of file From 9c76c7912c474ec9e59b6e84c56db8c1a6709891 Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 20 Apr 2023 20:49:44 -0500 Subject: [PATCH 04/14] rest framework added to apps --- nba_stats/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nba_stats/settings.py b/nba_stats/settings.py index 077df56..7f81aae 100644 --- a/nba_stats/settings.py +++ b/nba_stats/settings.py @@ -43,6 +43,7 @@ INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'nba_data', + 'rest_framework', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', From 9a5230759e85f008006a804e4d25a8828f6e40d9 Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 20 Apr 2023 20:53:25 -0500 Subject: [PATCH 05/14] create serializer --- nba_data/serializers.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 nba_data/serializers.py diff --git a/nba_data/serializers.py b/nba_data/serializers.py new file mode 100644 index 0000000..88634de --- /dev/null +++ b/nba_data/serializers.py @@ -0,0 +1,8 @@ +# Serializers are defined that will convert model instances to JSON +from rest_framework import serializers +from .models import PlayerData + +class PlayerDataSerializer(serializers.ModelSerializer): + class Meta: + model = PlayerData + fields = '__all__' From f17ea9920632a03a36e8d49f740c075fc025d93c Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 20 Apr 2023 20:59:04 -0500 Subject: [PATCH 06/14] remove old view and add class for new view for all players --- nba_data/views.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nba_data/views.py b/nba_data/views.py index 10f8362..0f2c6a3 100644 --- a/nba_data/views.py +++ b/nba_data/views.py @@ -2,9 +2,13 @@ from django.shortcuts import render from django.http import JsonResponse from django.core import serializers from .models import PlayerData +from rest_framework import generics +from .serializers import PlayerDataSerializer + # Create your views here. -def get_player_data(request): - data = serializers.serialize('json', PlayerData.objects.all()) - return JsonResponse(data, safe=False) \ No newline at end of file +# API view to list all PlayerData instances +class PlayerDataList(generics.ListCreateAPIView): + queryset = PlayerData.objects.all() + serializer_class = PlayerDataSerializer From ac45b1534aaeaefc84a0b561f87043e5774f096c Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 20 Apr 2023 21:01:42 -0500 Subject: [PATCH 07/14] add app url --- nba_data/urls.py | 4 ++-- nba_stats/urls.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nba_data/urls.py b/nba_data/urls.py index 100434e..38df167 100644 --- a/nba_data/urls.py +++ b/nba_data/urls.py @@ -1,6 +1,6 @@ from django.urls import path -from .views import get_player_data +from .views import PlayerDataList urlpatterns = [ - path('api/player_data/', get_player_data, name='player_data') + path('api/playerdata/', PlayerDataList.as_view(), name='playerdata_list'), ] \ No newline at end of file diff --git a/nba_stats/urls.py b/nba_stats/urls.py index 970cf52..a31a966 100644 --- a/nba_stats/urls.py +++ b/nba_stats/urls.py @@ -18,6 +18,6 @@ from django.contrib import admin from django.urls import path, include urlpatterns = [ - path('', include('nba_data.urls')), path('admin/', admin.site.urls), + path('', include('nba_data.urls')), ] From f39d19ea19ca3d8fa5fbab4f453d9702a9780ed5 Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 20 Apr 2023 21:10:38 -0500 Subject: [PATCH 08/14] data too big. will try pagination. --- nba_data/views.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nba_data/views.py b/nba_data/views.py index 0f2c6a3..5b3ee04 100644 --- a/nba_data/views.py +++ b/nba_data/views.py @@ -12,3 +12,9 @@ from .serializers import PlayerDataSerializer class PlayerDataList(generics.ListCreateAPIView): queryset = PlayerData.objects.all() serializer_class = PlayerDataSerializer + + +# Fetch player data by team + +class PlayerDataByTeamList(generics.ListAPIView): + From 7b38cb386eae48b2dc2ca447c236749b483011d2 Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 20 Apr 2023 21:24:19 -0500 Subject: [PATCH 09/14] fetch 100 players at a time w/pagination --- nba_data/views.py | 5 +---- nba_stats/settings.py | 6 ++++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/nba_data/views.py b/nba_data/views.py index 5b3ee04..82cf70f 100644 --- a/nba_data/views.py +++ b/nba_data/views.py @@ -12,9 +12,6 @@ from .serializers import PlayerDataSerializer class PlayerDataList(generics.ListCreateAPIView): queryset = PlayerData.objects.all() serializer_class = PlayerDataSerializer + # pagination_class = None -# Fetch player data by team - -class PlayerDataByTeamList(generics.ListAPIView): - diff --git a/nba_stats/settings.py b/nba_stats/settings.py index 7f81aae..eaeef38 100644 --- a/nba_stats/settings.py +++ b/nba_stats/settings.py @@ -136,3 +136,9 @@ STATIC_URL = 'static/' # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +REST_FRAMEWORK = { + 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', + 'PAGE_SIZE': 100, # Adjust this value according to the number of records you want to display per page +} + From a1e0443f7cd003a966f30effd461c016a121355f Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 20 Apr 2023 21:27:11 -0500 Subject: [PATCH 10/14] order by id --- nba_data/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nba_data/views.py b/nba_data/views.py index 82cf70f..b388f31 100644 --- a/nba_data/views.py +++ b/nba_data/views.py @@ -10,7 +10,7 @@ from .serializers import PlayerDataSerializer # API view to list all PlayerData instances class PlayerDataList(generics.ListCreateAPIView): - queryset = PlayerData.objects.all() + queryset = PlayerData.objects.all().order_by('id') serializer_class = PlayerDataSerializer # pagination_class = None From 135f3cbfb96aa61463fa5b02fda597d33713d39a Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Thu, 20 Apr 2023 21:50:59 -0500 Subject: [PATCH 11/14] add views for player name and season --- nba_data/urls.py | 5 ++++- nba_data/views.py | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/nba_data/urls.py b/nba_data/urls.py index 38df167..919d4ac 100644 --- a/nba_data/urls.py +++ b/nba_data/urls.py @@ -1,6 +1,9 @@ from django.urls import path -from .views import PlayerDataList +from .views import PlayerDataList, PlayerDataByTeamList, PlayerDataBySeasonList, PlayerDataByNameList urlpatterns = [ path('api/playerdata/', PlayerDataList.as_view(), name='playerdata_list'), + path('api/playerdata/team//', PlayerDataByTeamList.as_view(), name='playerdata_by_team'), + path('api/playerdata/season//', PlayerDataBySeasonList.as_view(), name='playerdata_by_season'), + path('api/playerdata/name//', PlayerDataByNameList.as_view(), name='playerdata_by_name'), ] \ No newline at end of file diff --git a/nba_data/views.py b/nba_data/views.py index b388f31..4ff8589 100644 --- a/nba_data/views.py +++ b/nba_data/views.py @@ -15,3 +15,28 @@ class PlayerDataList(generics.ListCreateAPIView): # pagination_class = None +# Fetch player data by team +class PlayerDataByTeamList(generics.ListAPIView): + serializer_class = PlayerDataSerializer + + def get_queryset(self): + team = self.kwargs['team'] + return PlayerData.objects.filter(team=team) + + +# Fetch player data by season +class PlayerDataBySeasonList(generics.ListAPIView): + serializer_class = PlayerDataSerializer + + def get_queryset(self): + season = self.kwargs['season'] + return PlayerData.objects.filter(season=season) + + +# Fetch player data by player name +class PlayerDataByNameList(generics.ListAPIView): + serializer_class = PlayerDataSerializer + + def get_queryset(self): + name = self.kwargs['name'] + return PlayerData.objects.filter(name__icontains=name) From c4ff08279ddb9929007c7d9efecfc0f5e448f2ee Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Fri, 21 Apr 2023 13:04:54 -0500 Subject: [PATCH 12/14] top 20 players route returned --- nba_data/urls.py | 3 ++- nba_data/views.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/nba_data/urls.py b/nba_data/urls.py index 919d4ac..dbdf7fc 100644 --- a/nba_data/urls.py +++ b/nba_data/urls.py @@ -1,9 +1,10 @@ from django.urls import path -from .views import PlayerDataList, PlayerDataByTeamList, PlayerDataBySeasonList, PlayerDataByNameList +from .views import PlayerDataList, PlayerDataByTeamList, PlayerDataBySeasonList, PlayerDataByNameList, TopScorersbySeasonList urlpatterns = [ path('api/playerdata/', PlayerDataList.as_view(), name='playerdata_list'), path('api/playerdata/team//', PlayerDataByTeamList.as_view(), name='playerdata_by_team'), path('api/playerdata/season//', PlayerDataBySeasonList.as_view(), name='playerdata_by_season'), path('api/playerdata/name//', PlayerDataByNameList.as_view(), name='playerdata_by_name'), + path('api/playerdata/topscorers/season//', TopScorersbySeasonList.as_view(), name='top_scorers_by_season'), ] \ No newline at end of file diff --git a/nba_data/views.py b/nba_data/views.py index 4ff8589..921f7a1 100644 --- a/nba_data/views.py +++ b/nba_data/views.py @@ -40,3 +40,13 @@ class PlayerDataByNameList(generics.ListAPIView): def get_queryset(self): name = self.kwargs['name'] return PlayerData.objects.filter(name__icontains=name) + + +# Fetch Top 20 players by PTS DESC for the season specified. +class TopScorersbySeasonList(generics.ListAPIView): + serializer_class = PlayerDataSerializer + + def get_queryset(self): + season = self.kwargs['season'] + return PlayerData.objects.filter(season=season).order_by('-PTS')[:20] + From b7720ca1386c203b04c78954ef43fdd885198719 Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Fri, 21 Apr 2023 20:24:36 -0500 Subject: [PATCH 13/14] server is running proper! --- nba_stats/settings.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nba_stats/settings.py b/nba_stats/settings.py index eaeef38..82a83cf 100644 --- a/nba_stats/settings.py +++ b/nba_stats/settings.py @@ -34,7 +34,7 @@ SECRET_KEY = SECRET_DJANGO # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = ['*'] # Application definition @@ -43,6 +43,7 @@ INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'nba_data', + 'corsheaders', 'rest_framework', 'django.contrib.contenttypes', 'django.contrib.sessions', @@ -51,6 +52,7 @@ INSTALLED_APPS = [ ] MIDDLEWARE = [ + 'corsheaders.middleware.CorsMiddleware', 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -142,3 +144,8 @@ REST_FRAMEWORK = { 'PAGE_SIZE': 100, # Adjust this value according to the number of records you want to display per page } +CORS_ALLOWED_ORIGINS = [ + "http://localhost:5173", # Replace this with the origin of your frontend app +] + + From 0be586a222497b79eda0a5571d1be53529be0ceb Mon Sep 17 00:00:00 2001 From: nprasad2077 <114438461+nprasad2077@users.noreply.github.com> Date: Fri, 21 Apr 2023 20:28:01 -0500 Subject: [PATCH 14/14] readme update --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 97fd164..386d558 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,15 @@   2. Backend API - -REST or GraphQL - - Flask (Python): Lightweight and easy to set up, suitable for small to medium projects. - Django (Python): A more comprehensive framework that includes an ORM and admin panel out-of-the-box. - - Express (Node.js): A fast and minimalistic web framework for Node.js. - - Ruby on Rails (Ruby): A powerful and mature full-stack web framework. -  + - Django REST Framework +  -3. Frontend +3. Backend DB + -Initially SQL lite, but it quickly met its limitations when trying to scrape data for 30 teams at once. It could not handle all the simulataneous connections. + - PostgreSQL is the new DB. + +4. Frontend - React/Redux - D3.js - Chart.js @@ -24,7 +25,7 @@ - Websockets or Server-Sent Events   -4. Deploy +1. Deploy