# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: bootcamp
#     language: python
#     name: bootcamp
# ---

# # Fetching Data
#
# This notebook includes code that fetches data from balldontlie API and then saves it as a csv file.

import requests
import pandas as pd
import time

pd.set_option('display.max_columns', None)


# NOTE: This make_request() function is different from the one in functions.py. It doesn't automatically detect if the request it a multi page request. (balldontlie API serves a maximum of 100 rows of data per request)

def make_request(endpoint, params=None, record_path=None, verbose=False):
    root = "https://www.balldontlie.io/api/v1/"
    response = requests.get(root + endpoint, params=params)
    if response.status_code != 200:
        print(response.status_code)
        return response
    if verbose: print("Success!")
    res = response.json()
    res = pd.json_normalize(res, record_path=record_path)
    return res


# + [markdown] tags=[]
# ### game data
#
# This data is only used for EDA purposes
# -

game_data = make_request("games", params={"page":1, "per_page":100, "seasons":[2017,2018,2019,2020]}, record_path="data", verbose=True)

# + jupyter={"outputs_hidden": true} tags=[]
# %%time
for i in range(2,501):
    if i%10 == 0: print(i)
    time.sleep(1.2)
    new_data = make_request("games", params={"page":i, "per_page":100, "seasons":[2017,2018,2019,2020]}, record_path="data")
    game_data = game_data.append(new_data)
# -

game_data.set_index("id", inplace=True)

game_data.to_csv("data/games.csv")

# + [markdown] tags=[]
# ### stats data
#
# This is the data used to build the model. Individual player stats for every NBA game since 1979.
#
# The first block of code is just to get the meta data. To see how many pages of data we are going to have to request (as balldontlie API serves a maximum of 100 rows of data per request.)

# + tags=[]
# there are over 11000 pages!
all_stats_data_meta = make_request("stats", params={"page":1, "per_page":100}, record_path=None)

all_stats_data_meta
# -

# This code gets all 11483 pages of data from the API and saves it in a dataframe

# + tags=[]
stats_data = pd.DataFrame()

for i in range(1, 11483):
    # Print what page we're on every 10 pages to keep track of progress
    if i % 10 == 0:
        print(i)
        
    # Make sure not to exceed 60 API requests per minute (balldontlie API is free but limits request per minite)
    time.sleep(1.1)
    
    # Make the request and append to the dataframe
    new_data = make_request("stats", params={"page":i, "per_page":100}, record_path="data")
    stats_data = stats_data.append(new_data)
print("Done!")
# -

# This code saves the data to a csv file. It's commented out to not accidentally overwrite the current csv file as it took hours to pull all the data.

# +
stats_data.set_index("id", inplace=True)

# stats_data.to_csv("data/stats_raw.csv")

# + jupyter={"source_hidden": true} tags=[]
for i in range(2, 11483):
    if i%100 == 0: print(i)
    time.sleep(1.5)
    new_data = make_request("stats", params={"page":i, "per_page":100}, record_path="data")
    all_stats_data = all_stats_data.append(new_data)
print("Done!")

StarcoderdataJupyterScriptsDedupFiltered