<NME> statcast.py
<BEF> import concurrent.futures
import warnings
import requests
import datetime
import warnings
import io

#TODO: does query work if end_dt befre start_dt? if not, detect this and swap them

from . import cache
from .utils import sanitize_date_range, statcast_date_range

_SC_SINGLE_GAME_REQUEST = "/statcast_search/csv?all=true&type=details&game_pk={game_pk}"
# pylint: disable=line-too-long
_SC_SMALL_REQUEST = "/statcast_search/csv?all=true&hfPT=&hfAB=&hfBBT=&hfPR=&hfZ=&stadium=&hfBBL=&hfNewZones=&hfGT=R%7CPO%7CS%7C=&hfSea=&hfSit=&player_type=pitcher&hfOuts=&opponent=&pitcher_throws=&batter_stands=&hfSA=&game_date_gt={start_dt}&game_date_lt={end_dt}&team={team}&position=&hfRO=&home_road=&hfFlag=&metric_1=&hfInn=&min_pitches=0&min_results=0&group_by=name&sort_col=pitches&player_event_sort=h_launch_speed&sort_order=desc&min_abs=0&type=details&"
# _MAX_SC_RESULTS = 40000

class StatcastException(Exception):
    pass

@cache.df_cache(expires=365)
def _small_request(start_dt: date, end_dt: date, team: Optional[str] = None) -> pd.DataFrame:
    data = statcast_ds.get_statcast_data_from_csv_url(
        _SC_SMALL_REQUEST.format(start_dt=str(start_dt), end_dt=str(end_dt), team=team if team else '')
    )
    if data is not None and not data.empty:
        if 'error' in data.columns:
            raise StatcastException(data['error'].values[0])

        data = data.sort_values(
            ['game_date', 'game_pk', 'at_bat_number', 'pitch_number'],
            ascending=False
        )

def small_request(start_dt,end_dt):
    url = "https://baseballsavant.mlb.com/statcast_search/csv?all=true&hfPT=&hfAB=&hfBBT=&hfPR=&hfZ=&stadium=&hfBBL=&hfNewZones=&hfGT=R%7CPO%7CS%7C=&hfSea=&hfSit=&player_type=pitcher&hfOuts=&opponent=&pitcher_throws=&batter_stands=&hfSA=&game_date_gt={}&game_date_lt={}&team=&position=&hfRO=&home_road=&hfFlag=&metric_1=&hfInn=&min_pitches=0&min_results=0&group_by=name&sort_col=pitches&player_event_sort=h_launch_speed&sort_order=desc&min_abs=0&type=details&".format(start_dt, end_dt)
    s=requests.get(url, timeout=None).content
    data = pd.read_csv(io.StringIO(s.decode('utf-8')))
    return data

def large_request(start_dt,end_dt,d1,d2,step):
gremlins; computer repair by associates of Rudy Giuliani; electromagnetic interference from metal trash cans; etc.;
you could lose a lot of progress. Enabling caching will allow you to immediately recover all the successful
subqueries if that happens.'''


def _check_warning(start_dt: date, end_dt: date) -> None:
    if not cache.config.enabled and (end_dt - start_dt).days >= 42:
        warnings.warn(_OVERSIZE_WARNING)
    #step = 3 # number of days per mini-query (test this later to see how large I can make this without losing data)
    d = d1 + datetime.timedelta(days=step)
    while d <= d2: #while intermediate query end_dt <= global query end_dt, keep looping
        start_dt = d1.strftime('%Y-%m-%d')
        intermediate_end_dt = d.strftime('%Y-%m-%d')
        data = small_request(start_dt,intermediate_end_dt)
        # append to list of dataframes
        dataframe_list.append(data)
        print("Completed sub-query from {} to {}".format(start_dt,intermediate_end_dt))
        # increment dates
        d1 = d1 + datetime.timedelta(days=step+1)
        d = d + datetime.timedelta(days=step+1)

    # if start date > end date after being incremented, the loop captured each date's data
            for subq_start, subq_end in date_range:
                dataframe_list.append(_small_request(subq_start, subq_end, team=team))
                progress.update(1)

    # Concatenate all dataframes into final result set
    if dataframe_list:
        final_data = pd.concat(dataframe_list, axis=0).convert_dtypes(convert_string=False)
        final_data = final_data.sort_values(
            ['game_date', 'game_pk', 'at_bat_number', 'pitch_number'],
            ascending=False
        )
    else:
        final_data = pd.DataFrame()
    return final_data


def statcast(start_dt: str = None, end_dt: str = None, team: str = None,
             verbose: bool = True, parallel: bool = True) -> pd.DataFrame:
    """
    Pulls statcast play-level data from Baseball Savant for a given date range.

    INPUTS:
    start_dt: YYYY-MM-DD : the first date for which you want statcast data
    end_dt: YYYY-MM-DD : the last date for which you want statcast data
    team: optional (defaults to None) : city abbreviation of the team you want data for (e.g. SEA or BOS)
    verbose: bool (defaults to True) : whether to print updates on query progress
    parallel: bool (defaults to True) : whether to parallelize HTTP requests in large queries

    If no arguments are provided, this will return yesterday's statcast data.
    If one date is provided, it will return that date's statcast data.
    """

    start_dt_date, end_dt_date = sanitize_date_range(start_dt, end_dt)

    return _handle_request(start_dt_date, end_dt_date, 1, verbose=verbose,
                           team=team, parallel=parallel)


def statcast_single_game(game_pk: Union[str, int]) -> pd.DataFrame:
    """
    Pulls statcast play-level data from Baseball Savant for a single game,
    identified by its MLB game ID (game_pk in statcast data)

    INPUTS:
    game_pk : 6-digit integer MLB game ID to retrieve
    """

    data = statcast_ds.get_statcast_data_from_csv_url(
        _SC_SINGLE_GAME_REQUEST.format(game_pk=game_pk)
    )

    if data is None or data.empty:
        return None

    if 'error' in data.columns:
        raise StatcastException(data['error'].values[0])

    return data.sort_values(
        ['game_date', 'game_pk', 'at_bat_number', 'pitch_number'],
        ascending=False
    )

<MSG> Make statcast scrapes work across multiple seasons and skip offseason dates

<DFF> @@ -3,6 +3,7 @@ import pandas as pd
 import requests
 import datetime
 import warnings
+#import csv
 import io
 
 #TODO: does query work if end_dt befre start_dt? if not, detect this and swap them
@@ -35,7 +36,7 @@ def sanitize_input(start_dt, end_dt):
 def small_request(start_dt,end_dt):
     url = "https://baseballsavant.mlb.com/statcast_search/csv?all=true&hfPT=&hfAB=&hfBBT=&hfPR=&hfZ=&stadium=&hfBBL=&hfNewZones=&hfGT=R%7CPO%7CS%7C=&hfSea=&hfSit=&player_type=pitcher&hfOuts=&opponent=&pitcher_throws=&batter_stands=&hfSA=&game_date_gt={}&game_date_lt={}&team=&position=&hfRO=&home_road=&hfFlag=&metric_1=&hfInn=&min_pitches=0&min_results=0&group_by=name&sort_col=pitches&player_event_sort=h_launch_speed&sort_order=desc&min_abs=0&type=details&".format(start_dt, end_dt)
     s=requests.get(url, timeout=None).content
-    data = pd.read_csv(io.StringIO(s.decode('utf-8')))
+    data = pd.read_csv(io.StringIO(s.decode('utf-8')))#, error_bad_lines=False) # skips 'bad lines' breaking scrapes. still testing this.
     return data
 
 def large_request(start_dt,end_dt,d1,d2,step):
@@ -50,14 +51,28 @@ def large_request(start_dt,end_dt,d1,d2,step):
     #step = 3 # number of days per mini-query (test this later to see how large I can make this without losing data)
     d = d1 + datetime.timedelta(days=step)
     while d <= d2: #while intermediate query end_dt <= global query end_dt, keep looping
+        # dates before 3/15 and after 11/15 will always be offseason
+        # if these dates are detected, check if the next season is within the user's query
+        # if yes, fast-forward to the next season to avoid empty requests
+        # if no, break the loop. all useful data has been pulled.
+        if ((d.month < 4 and d.day < 15) or (d1.month > 10 and d1.day > 14)):
+            if d2.year > d.year:
+                print('Skipping offseason dates')
+                d1 = d1.replace(month=3,day=15,year=d1.year+1)
+                d = d1 + datetime.timedelta(days=step+1)
+            else:
+                break
+
         start_dt = d1.strftime('%Y-%m-%d')
         intermediate_end_dt = d.strftime('%Y-%m-%d')
         data = small_request(start_dt,intermediate_end_dt)
-        # append to list of dataframes
-        dataframe_list.append(data)
+        # append to list of dataframes if not empty
+        if data.shape[0] > 0:
+            dataframe_list.append(data)
+
         print("Completed sub-query from {} to {}".format(start_dt,intermediate_end_dt))
         # increment dates
-        d1 = d1 + datetime.timedelta(days=step+1)
+        d1 = d + datetime.timedelta(days=1)
         d = d + datetime.timedelta(days=step+1)
 
     # if start date > end date after being incremented, the loop captured each date's data

{'addition_count': 19, 'commit_subject': 'Make statcast scrapes work across multiple seasons and skip offseason dates', 'deletion_count': 4, 'file_extension': '.py', 'lang': 'py', 'license': 'mit', 'repo_name': 'jldbc/pybaseball'}