# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: conda_python3
#     language: python
#     name: conda_python3
# ---

# + colab={} colab_type="code" id="ir7ZCnFzxZIx"
# #!pip install psycopg2-binary

# + colab={"base_uri": "https://localhost:8080/", "height": 310} colab_type="code" id="gfmn3oGRHvqS" outputId="f0ae607e-1683-415d-905a-1cb5e332de3c"
# #!pip install category_encoders

# +
# #!pip install catboost

# + colab={"base_uri": "https://localhost:8080/", "height": 145} colab_type="code" id="9KA_kRzkJUCz" outputId="603d1e1b-26a1-4bd6-9798-b1447fbed280"
# #!git clone https://github.com/labs15-baseball-pitch-predictor/RC-v1.1--Data-Cleaning-and-Feature-Engineering.git

# + colab={} colab_type="code" id="eyJcr69s4WnJ"
import pandas as pd
import numpy as np
import random
import pickle
import gc
import psycopg2
#import category_encoders as ce

from sklearn.preprocessing import RobustScaler, StandardScaler
pd.set_option('display.max_columns', 500)
pd.set_option('display.max_rows', 500)
# -

pwd

# + [markdown] colab_type="text" id="i0XdqAkP4WnW"
# ## Load in the pre-pitcher preprocessed_df
#
#

# + colab={} colab_type="code" id="N1KYonGe4WnZ"
#load in pickled preprocessed df
path1 = '/home/ec2-user/SageMaker/RC-v1.1--Data-Cleaning-and-Feature-Engineering/data/pre_pitcher_pre_processed_df_part1.pkl'
path2 = '/home/ec2-user/SageMaker/RC-v1.1--Data-Cleaning-and-Feature-Engineering/data/pre_pitcher_pre_processed_df_part2.pkl'
df1 = pd.read_pickle(path1, compression='zip')
df2 = pd.read_pickle(path2, compression='zip')

df = pd.concat([df1, df2])

# + colab={"base_uri": "https://localhost:8080/", "height": 35} colab_type="code" id="7EkbHQ4w4Wne" outputId="86247c57-fdec-4123-f8a3-716e5b9776bd"
#clear some memory:
del df1
del df2
gc.collect()


# + colab={} colab_type="code" id="APdo1GMw4Wns"
#make list of pitchers (for filling in Nans in batting stat later for pitcher slot in batting order)

def make_pitcher_list(pre_processed_df):
    #make list of all the pitcher ids in the preprocessed_df
    pitcher_list = df['pitcher'].unique().tolist()
    
    #on rare occassions in extra-inning games, a position player is called in to pitch: 
    #to avoid categorizing him as a pitcher we'll only count pitchers with over 100 pitches:
    for pitcher in pitcher_list:
        if len(df[df['pitcher'] == pitcher]) < 100:
            pitcher_list.remove(pitcher)
    return pitcher_list

pitcher_list = make_pitcher_list(df)

# + [markdown] colab_type="text" id="Frq1Z-F-4Wnk"
# ## Choose the pitcher to model

# + colab={} colab_type="code" id="0umVTRXw58YJ"
#df.groupby('pitcher').count().sort_values('pitch_type', ascending=False).head(10)

# + colab={"base_uri": "https://localhost:8080/", "height": 460} colab_type="code" id="VfUOyQvf4Wnm" outputId="0a347a05-d630-4944-f2ca-ad847074d7ab"
#dictionary mapping top 10 pitchers with most pitches name with pitcher id

pitcher_id_dict = {'<NAME>': 434378.0, '<NAME>': 545333.0, 
                   '<NAME>': 605400.0, '<NAME>': 458681.0, '<NAME>':
                   543037.0, '<NAME>':594798.0, '<NAME>':425844.0,
                   '<NAME>': 571578.0, '<NAME>': 453286.0,
                   '<NAME>': 502043.0}

#make pitcher_df subset where pitcher == picther_id
pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']

pitcher_df = df[df['pitcher'] == pitcher_id]
pitcher_df.head()

# + [markdown] colab_type="text" id="TKKLXIep4Wny"
# ## Add the pitcher prior tendencies (Pitcher Scouting Report)

# + colab={"base_uri": "https://localhost:8080/", "height": 54} colab_type="code" id="3x8fsuVwHQ5E" outputId="c4a87802-e0b1-4297-a555-2ea6ac028975"
# %%time
#read in 2017 pre_procesed step 1 pickle and filter for pitcher_id

df_17_path = '/home/ec2-user/SageMaker/RC-v1.1--Data-Cleaning-and-Feature-Engineering/data/preprocessed_2017.pkl'
df17 = pd.read_pickle(df_17_path, compression = 'zip')
pitcher_df17 = df17[df17['pitcher'] == pitcher_id]


# + colab={} colab_type="code" id="K2ufyTDb_C2J"
def get_left_right_pitch_tendencies(pitcher_df):
    #split the df into left hand and right handed batters
    left = pitcher_df[pitcher_df['stand'] == 'L'].copy()
    right = pitcher_df[pitcher_df['stand'] == 'R'].copy()
    
    #assign the normalized value counts for this pitchers pitch types to a dictionary
    overall_left = left['pitch_cat'].value_counts(normalize=True).to_dict()
    overall_right = right['pitch_cat'].value_counts(normalize=True).to_dict()
    
    #initialize empty dict for count categories tendencies
    by_count_left = {}
    by_count_right = {}
    
    #loop over each count category and get the pitchers tendencies and add to the dict
    for cat in pitcher_df['count_cat'].unique().tolist():
        left_subset = left[left['count_cat'] == cat]
        right_subset = right[right['count_cat'] == cat]
        
        by_count_left[cat] = left_subset['pitch_cat'].value_counts(normalize=True).to_dict()
        by_count_right[cat] = right_subset['pitch_cat'].value_counts(normalize=True).to_dict()
        
    return overall_left, overall_right, by_count_left, by_count_right

def make_tendency_features(pitcher_df, overall_left, overall_right, by_count_left, by_count_right):
    #helper functions to vectorize w/ df.apply():
    def get_overall_left_perc(x):
        return overall_left[x] * 100
    def get_overall_right_perc(x):
        return overall_right[x] * 100
    def get_by_count_left_perc(x):
        try:
            return by_count_left[x][pitch_type] * 100
        except KeyError:
            return 0
    def get_by_count_right_perc(x):
        try:
            return by_count_right[x][pitch_type] * 100
        except KeyError:
            return 0
    
    left = pitcher_df[pitcher_df['stand'] == 'L'].copy()
    right = pitcher_df[pitcher_df['stand'] == 'R'].copy()
    
    pitch_types_left = overall_left.keys()
    pitch_types_right = overall_right.keys()
    
    #Left
    for pitch_type in pitch_types_left:
        overall_feature = 'overall_' + pitch_type + '_perc'
        count_cat_feature = 'count_cat_' + pitch_type + '_perc'
      
        left[overall_feature] = pitch_type
        left[overall_feature] = left[overall_feature].apply(get_overall_left_perc)
        left[count_cat_feature] = left['count_cat'].apply(get_by_count_left_perc)
    #Right
    for pitch_type in pitch_types_right:
        overall_feature = 'overall_' + pitch_type + '_perc'
        count_cat_feature = 'count_cat_' + pitch_type + '_perc'
      
        right[overall_feature] = pitch_type
        right[overall_feature] = right[overall_feature].apply(get_overall_right_perc)
        right[count_cat_feature] = right['count_cat'].apply(get_by_count_right_perc)
        
    
    return pd.concat([left,right], sort=False).sort_values(by=['game_date', 'game_pk', 'at_bat_number', 'pitch_number'])

def add_pitcher_scouting_report(pitcher_df, pitcher_df17, start_dates, end_dates):
    df = pd.concat([pitcher_df, pitcher_df17], sort=False)
    
    #initialize empty list to store dfs (concat them together later)
    df_list = []
    
    #iterate over each period
    for i in range(len(start_dates)):
      
        #make the prior and current dfs:
        prior_df = df[df['game_date'] < start_dates[i]]
        current_df = df[(df['game_date'] >= start_dates[i]) & (df['game_date'] <= end_dates[i])].copy()
        
        #get the pitch tendencies from prior:
        overall_left, overall_right, by_count_left, by_count_right = get_left_right_pitch_tendencies(prior_df)
        
        #make the pitch tendencies features on current:
        current_df = make_tendency_features(current_df, overall_left, overall_right, by_count_left, by_count_right)
        
        #append the df to the list
        df_list.append(current_df)
    
    df = pd.concat(df_list, sort=False)
    return df


# + colab={} colab_type="code" id="iHaTqfVGdrAT"
start_dates = ['2018-03-29', '2018-05-01', '2018-06-01', '2018-07-01', '2018-08-01', 
               '2018-09-01', '2019-03-28', '2019-05-01', '2019-06-01', '2019-07-01', 
               '2019-08-01']

end_dates =  ['2018-04-30', '2018-05-31', '2018-06-30', '2018-07-31', '2018-08-31', 
              '2018-10-01', '2019-04-30', '2019-05-31', '2019-06-30', '2019-07-31', 
              '2019-08-31']

# + colab={"base_uri": "https://localhost:8080/", "height": 54} colab_type="code" id="Rjt6OMtlEnlM" outputId="71aed053-79cf-4d2d-821c-8e4ff329e5c5"
# %%time
pitcher_df = add_pitcher_scouting_report(pitcher_df, pitcher_df17, start_dates, end_dates)

# + colab={"base_uri": "https://localhost:8080/", "height": 460} colab_type="code" id="kvHUuXk_FZ4U" outputId="3a008f2e-0dc7-4a42-f008-4d4008e2e176"
pitcher_df.head()
# -

pitcher_df.description.value_counts()

# + [markdown] colab_type="text" id="MctnVBNt4WoF"
# ## Add Game Features:
#
# #### Batting Order, Game pitch_count, & Trailing Pitch Features
# -

pitcher_df.type.head()


# + colab={} colab_type="code" id="jD2hVjOo4WoN"
def make_game_batting_order(game_df):
    game_df = game_df.sort_values(by=['at_bat_number', 'pitch_number'])
    all_batters = game_df['batter'].unique().tolist()
    #re-set the at_bat_number for the game to be sequential starting at 1
    at_bat_keys = game_df['at_bat_number'].unique().tolist()
    at_bat_values = range(1, len(at_bat_keys)+1)
    at_bat_map = dict(zip(at_bat_keys, at_bat_values))
    game_df['at_bat_number'] = game_df['at_bat_number'].replace(at_bat_map)
    
    #get the first 9 batter ids
    first_9_batter_subset = game_df[game_df['at_bat_number'] < 10]
    first_9_batters = first_9_batter_subset['batter'].unique().tolist()
    
    #map the batter id to batting order position 1-9
    batting_order_map = dict(zip(first_9_batters, range(1,10)))
    
    #for anyone else who bats later in the game, assign 0 (pinch hitter) to their batting order slot
    other_batters = list(set(all_batters) - set(first_9_batters))
    if len(other_batters) > 0:
        for batter in other_batters:
            batting_order_map[batter] = 0
    try:
        game_df['batting_order_slot'] = game_df['batter'].apply(lambda x: batting_order_map[x])    
    except KeyError:
        game_df = None
        return game_df
    
    game_df['pitcher_AB'] = game_df['batter'].apply(lambda x: 1 if x in pitcher_list else 0)
    game_df['batting_order_slot'] = game_df['batting_order_slot'].where(game_df['pitcher_AB'] == 0, other=10)
    return game_df

def get_pitch_tendencies(pitcher_df):
    #assign the normalized value counts for this pitchers pitch types to a dictionary
    pitcher_tendencies_overall = pitcher_df['pitch_cat'].value_counts(normalize=True).to_dict()

    #initialize empty dict for count categories tendencies
    pitcher_tendencies_by_count = {}
    
    #loop over each count category and get the pitchers tendencies and add to the dict
    for cat in pitcher_df['count_cat'].unique().tolist():
        subset = pitcher_df[pitcher_df['count_cat'] == cat]
        pitcher_tendencies_by_count[cat] = subset['pitch_cat'].value_counts(normalize=True).to_dict()
    return pitcher_tendencies_overall, pitcher_tendencies_by_count

def make_game_pitchcount_and_trailing_pitch_features_and_batting_order(pitcher_df, pitcher_list):
    df = pitcher_df.copy()
    all_games = []
    
    print('#pitches in df before: ' + str(len(df)))
    
    pitcher_tendencies_overall, pitcher_tendencies_by_count = get_pitch_tendencies(df)
    games = df['game_pk'].unique().tolist()
    
    for game in games:
        
        #take the first game and make the pitch count feature
        game_df = df[df['game_pk'] == game].copy()
        game_df['pitch_count'] = range(1, game_df.shape[0] + 1)
    
        #make the L1 pitch features:
        game_df['L1_pitch_type'] = game_df['pitch_type'].shift(periods=1)
        game_df['L1_pitch_result'] = game_df['type'].shift(periods=1)
        game_df['L1_description'] = game_df['description'].shift(periods=1)
        game_df['L1_pitch_zone'] = game_df['zone'].shift(periods=1)
        game_df['L1_batter_swung'] = game_df['batter_swung'].shift(periods=1)
        game_df['L1_chased'] = game_df['chased'].shift(periods=1)        
        
        #add L2 and L3 features:
        
        game_df['L2_pitch_type'] = game_df['pitch_type'].shift(periods=2)
        game_df['L2_pitch_result'] = game_df['type'].shift(periods=2)
        game_df['L2_description'] = game_df['description'].shift(periods=2)
        game_df['L2_pitch_zone'] = game_df['zone'].shift(periods=2)
        game_df['L2_chased'] = game_df['chased'].shift(periods=2)
        
        game_df['L3_pitch_type'] = game_df['pitch_type'].shift(periods=3)
        game_df['L3_description'] = game_df['description'].shift(periods=3)
        game_df['L3_pitch_zone'] = game_df['zone'].shift(periods=3)
        game_df['L3_chased'] = game_df['chased'].shift(periods=3)
        
        str_shifted_cols = ['L1_pitch_type', 'L1_pitch_result','L1_description', 'L2_pitch_type', 
                            'L2_pitch_result','L2_description', 'L3_pitch_type','L3_description']
        
        num_shifted_cols = ['L1_batter_swung', 'L1_chased', 'L1_pitch_zone', 'L2_chased',
                            'L2_pitch_zone', 'L3_chased', 'L3_pitch_zone']
        
        #fill Nan for shifted cols w/ string N/A
        game_df[str_shifted_cols] = game_df[str_shifted_cols].replace({np.nan:'N/A'})
        game_df[num_shifted_cols] = game_df[num_shifted_cols].replace({np.nan:-1})
        
    
        #overall strike % (to fill in for first 5 pitches L5_strike_perc)
        overall_strike_perc = df['type'].value_counts(normalize=True)['S'] * 100

        #make the trailing 5 pitches:
        for index, row in game_df.iterrows():
            #for the first 5 rows, use overall pitcher tendencies    
            if row['pitch_count'] < 6:
                #fill with overall tendencies
                for pitch in list(pitcher_tendencies_overall.keys()):
                    feature = 'L5_' + pitch + '_perc'
                    game_df.at[index, feature] = pitcher_tendencies_overall[pitch] * 100
                    feature = 'L15_' + pitch + '_perc'
                    game_df.at[index, feature] = pitcher_tendencies_overall[pitch] * 100
                #strike %
                game_df.at[index, 'L5_strike_perc'] = overall_strike_perc
                game_df.at[index, 'L15_strike_perc'] = overall_strike_perc

            else:
                current_pitch = game_df.at[index, 'pitch_count']
                #make a subset of the prev 5 pitches
                subset = game_df[(game_df['pitch_count'] > current_pitch - 6) & (game_df['pitch_count'] < current_pitch)]
                #grab the value count percentages for the last 5 pitches
                subset_percentages = subset['pitch_cat'].value_counts(normalize=True).to_dict()
                try:
                    L5_strike_perc = subset['type'].value_counts(normalize=True)['S'] * 100
                except KeyError:
                    L5_strike_perc = 0

                game_df.at[index, 'L5_strike_perc'] = L5_strike_perc

                #iterate over all possible pitch types this pitcher throws:
                for pitch in list(pitcher_tendencies_overall.keys()):
                    feature = 'L5_' + pitch + '_perc'
                    #if he has thrown that pitch type in last 5
                    try:
                        game_df.at[index, feature] = subset_percentages[pitch] * 100
                    #except for when he hasnt thrown that type in last 5
                    except:
                        game_df.at[index, feature] = 0

                if row['pitch_count'] < 16:
                    #make a subset of the prev 15 pitches
                    subset = game_df[(game_df['pitch_count'] < current_pitch)]
                    #grab the value count percentages for the last 15 pitches
                    subset_percentages = subset['pitch_cat'].value_counts(normalize=True).to_dict()
                    try:
                        L15_strike_perc = subset['type'].value_counts(normalize=True)['S'] * 100
                    except KeyError:
                        L15_strike_perc = 0

                    game_df.at[index, 'L15_strike_perc'] = L15_strike_perc

                    #iterate over all possible pitch types this pitcher throws:
                    for pitch in list(pitcher_tendencies_overall.keys()):
                        feature = 'L15_' + pitch + '_perc'
                        #if he has thrown that pitch type in last 15
                        try:
                            game_df.at[index, feature] = subset_percentages[pitch] * 100
                        #except for when he hasnt thrown that type in last 5
                        except:
                            game_df.at[index, feature] = 0
                else:
                    #make a subset of the prev 15 pitches
                    subset = game_df[(game_df['pitch_count'] > current_pitch - 16) & (game_df['pitch_count'] < current_pitch)]
                    #grab the value count percentages for the last 5 pitches
                    subset_percentages = subset['pitch_cat'].value_counts(normalize=True).to_dict()
                    try:
                        L15_strike_perc = subset['type'].value_counts(normalize=True)['S'] * 100
                    except KeyError:
                        L15_strike_perc = 0

                    game_df.at[index, 'L15_strike_perc'] = L15_strike_perc

                    #iterate over all possible pitch types this pitcher throws:
                    for pitch in list(pitcher_tendencies_overall.keys()):
                        feature = 'L15_' + pitch + '_perc'
                        #if he has thrown that pitch type in last 5
                        try:
                            game_df.at[index, feature] = subset_percentages[pitch] * 100
                        #except for when he hasnt thrown that type in last 5
                        except:
                            game_df.at[index, feature] = 0
                            
        #apply the battting order features to the game:                
        game_df = make_game_batting_order(game_df)
    
        all_games.append(game_df)
    
    new_df = pd.concat(all_games).sort_values(by=['game_date', 'game_pk', 'at_bat_number', 'pitch_number'])
    
    print('# pitches in df after: ' + str(len(new_df)))
    
    return new_df

def make_prev_ab_walk_basehit_run_and_homerun_features(pitcher_df):
    
    all_games = []
    #iterate over each game
    for game in pitcher_df['game_pk'].unique():
        #make subset df for that game
        game_df = pitcher_df[pitcher_df['game_pk'] == game].copy()
        #initialize columns to False:
        game_df['prev_ab_run_scored'] = 0
        game_df['prev_ab_homerun'] = 0
        game_df['prev_ab_walk'] = 0
        game_df['prev_ab_basehit'] = 0
        game_df['prev_ab_strikeout'] = 0
        
        #this gets the 
        at_bats = game_df['at_bat_number'].sort_values().unique()
        
        #initialize empty dicts
        run_scored = []
        homeruns = []
        walks = []
        basehits = []
        strikeouts = []
        
        walks = ['walk', 'hit_by_pitch']
        basehits = ['single', 'double', 'triple', 'home_run']
        
        #starting w/ 2nd AB, iterate thru to the end of the at_bats:
        for ab in at_bats[2:]:
            #get the index for the last pitch of the prev AB
            prev_ab_last_pitch_index = game_df[game_df['at_bat_number'] == ab-1]['pitch_number'].index.max()
            #check if the last pitch resulted in a walk or hit by pitch:
            if game_df.loc[prev_ab_last_pitch_index]['events'] in walks:
                #if so, add an entry
                walks.append(ab)
            #check if last pitch gave up a basehit:
            elif game_df.loc[prev_ab_last_pitch_index]['events'] in basehits:
                basehits.append(ab)
            elif game_df.loc[prev_ab_last_pitch_index]['events'] == 'strikeout':
                strikeouts.append(ab)
            
            #to check if prev AB resulted in a run scoring: compare score before and after the AB
            prev_score = game_df[game_df['at_bat_number'] == ab-1]['bat_score'].values[0]
            current_score = game_df[game_df['at_bat_number'] == ab]['bat_score'].values[0]
            
            if current_score > prev_score:
                run_scored.append(ab)
                 
                #check if last AB gave up a homerun:
                if game_df.loc[prev_ab_last_pitch_index]['events'] == 'home_run':
                    homeruns.append(ab)
                    
        #iterate over each at_bat, and add the features to the df where appropriate
        for ab in at_bats:
            idx = game_df[game_df['at_bat_number'] == ab].index
            if ab in walks:
                game_df.at[idx, 'prev_ab_walk'] = 1
            elif ab in basehits:
                game_df.at[idx, 'prev_ab_basehit'] = 1
            elif ab in strikeouts:
                game_df.at[idx, 'prev_ab_strikeout'] = 1
            if ab in run_scored:
                game_df.at[idx, 'prev_ab_run_scored'] = 1
                if ab in homeruns:
                    game_df.at[idx, 'prev_ab_homerun'] = 1
        all_games.append(game_df)
        
    return pd.concat(all_games).sort_values(by=['game_date', 'game_pk', 'pitch_count'])


# + colab={"base_uri": "https://localhost:8080/", "height": 90} colab_type="code" id="6IzMGnTeBQOu" outputId="9e8be00b-5235-4360-e8b5-fbcac16e9bc0"
# %%time
pitcher_df = make_game_pitchcount_and_trailing_pitch_features_and_batting_order(pitcher_df, pitcher_list)

# + colab={"base_uri": "https://localhost:8080/", "height": 54} colab_type="code" id="5OhK2lCnicYG" outputId="fb7d2fa9-e484-4040-dab9-7fdfe9db036c"
# %%time
pitcher_df = make_prev_ab_walk_basehit_run_and_homerun_features(pitcher_df)
# -

pitcher_df[['description', 'type', 'events','des', 'batter_swung', 'chased']].head(10)

# + colab={"base_uri": "https://localhost:8080/", "height": 668} colab_type="code" id="4XceZOp5DvhK" outputId="b3f4852a-3ada-4c56-d9d3-32d99bb940d4"
cols = ['pitch_count', 'pitch_type', 'type', 'batting_order_slot']

for col in pitcher_df.columns:
    if col[:1] == 'L':
        cols.append(col)
    if col[-4:] == 'perc':
        if col[:2]=='L5':
            cols.append(col)
        if col[:3]=='L15':
            cols.append(col)
        
pitcher_df[cols].head(20)

# + colab={"base_uri": "https://localhost:8080/", "height": 668} colab_type="code" id="1kqaBj-RMgyX" outputId="4664620a-35c4-4825-d476-bc86ef6f2aca"
#pitcher_df[cols].tail(20)

# + [markdown] colab_type="text" id="3Jg6Bijx4Wob"
# ## Fill Batting Scouting Report NaNs
#
# #### Load in the batting_order_slot map pickle

# + colab={} colab_type="code" id="JcC6MIdY4Wod"
batting_path = '/home/ec2-user/SageMaker/RC-v1.1--Data-Cleaning-and-Feature-Engineering/data/batting_order_slot_map.pkl'
batting_order_slot_map = pickle.load(open(batting_path, "rb"))
# -

batting_order_slot_map[10] = batting_order_slot_map['pitcher']
batting_order_slot_map[0] = batting_order_slot_map['PH']

batting_order_slot_map[0]

# + colab={} colab_type="code" id="qUxEzWra4Wol"
batter_cols = ['fastball_perc_faced','fastball_chase_perc','fastball_bip_swung_perc', 'fastball_taken_strike_perc',
               'fastball_est_woba', 'fastball_babip', 'fastball_iso_value', 'breaking_perc_faced', 'breaking_chase_perc',
               'breaking_bip_swung_perc', 'breaking_taken_strike_perc', 'breaking_est_woba', 'breaking_babip', 
               'breaking_iso_value', 'offspeed_perc_faced', 'offspeed_chase_perc', 'offspeed_bip_swung_perc',
               'offspeed_taken_strike_perc', 'offspeed_est_woba', 'offspeed_babip', 'offspeed_iso_value',
               'pitchout_perc_faced']

def fill_batting_nans(pitcher_df, batting_order_slot_map):
    df = pitcher_df.copy()
    for slot in df['batting_order_slot'].unique().tolist():
        subset = df[df['batting_order_slot'] == slot].copy()
        df = df.drop(subset.index)
        for col in batter_cols:
            subset[col] = subset[col].fillna(batting_order_slot_map[slot][col])
        df = pd.concat([df, subset])
        print('finished w/ slot: ' + str(slot))
    df = df.sort_values(by=['game_date', 'game_pk', 'pitch_count'])
    return df


# + colab={"base_uri": "https://localhost:8080/", "height": 254} colab_type="code" id="4cJXqXKk4Wos" outputId="3a9b0204-5406-4a69-befe-7e7bae0ed6e9"
# %%time
pitcher_df = fill_batting_nans(pitcher_df, batting_order_slot_map)


# + [markdown] colab_type="text" id="44EuFvIg4Wox"
# ## Add Pitcher / Batter Matchup History Report

# + colab={} colab_type="code" id="CJIMfsZ3xhdi"
def add_pb_matchup_priors(pitcher_df, pitcher_df17, start_dates, end_dates):
    df = pd.concat([pitcher_df, pitcher_df17], sort=False)
    
    #initialize empty list to store dfs (concat them together later)
    df_list = []
    
    #iterate over each period
    for i in range(len(start_dates)):
        
        #make the prior and current dfs:
        prior_df = df[df['game_date'] < start_dates[i]]
        current_df = df[(df['game_date'] >= start_dates[i]) & (df['game_date'] <= end_dates[i])]
        
        #get all the pitch_types this pitcher has thrown in the past:
        pitch_types = prior_df['pitch_cat'].unique().tolist()
        
        try:
            pitch_types.remove('PO')
        except:
            pass
        
        #get a list of the batters in the current_df
        current_batters = current_df['batter'].unique().tolist()
        
        batters_dict = {}
        
        current_df_list = []
        
        for batter in current_batters:
            batter_df_list = []
            
            #first use subset from prior df
            batter_subset = prior_df[prior_df['batter'] == batter].copy()
            #if pitcher has never faced this batter before:
            if batter_subset.empty:
                #get the left or right handedness of the batter
                stand = current_df[current_df['batter'] == batter]['stand'].values[0]
                #use overall prior tendencies vs left or right handed hitters
                overall, by_count = get_pitch_tendencies(prior_df[prior_df['stand'] == stand])
            else:
                overall, by_count = get_pitch_tendencies(batter_subset)
            batters_dict[batter] = by_count
            
            #now use subset of current_df where batter=batter
            batter_subset = current_df[current_df['batter'] == batter].copy()
            #iterate over the different count_cat types:
            for count_cat in ['ahead', 'behind', 'neutral']:
                count_subset = batter_subset[batter_subset['count_cat'] == count_cat].copy()
                if count_subset.empty:
                    continue
                else:
                    for pitch in pitch_types:
                        try:
                            count_subset['PB_'+pitch] = batters_dict[batter][count_cat][pitch] * 100
                        except KeyError:
                            count_subset['PB_'+pitch] = 0
                            
                current_df_list.append(count_subset)
        
        current_df = pd.concat(current_df_list, sort=False)
        df_list.append(current_df)
                    
    new_df = pd.concat(df_list, sort=False).sort_values(by=['game_date', 'game_pk', 'pitch_count'])
    return new_df


# + colab={"base_uri": "https://localhost:8080/", "height": 54} colab_type="code" id="wP4Jvcf56PjU" outputId="622ba356-dee0-4d40-8ae7-b7a51b882388"
# %%time
pitcher_df = add_pb_matchup_priors(pitcher_df, pitcher_df17, start_dates, end_dates)

# + colab={"base_uri": "https://localhost:8080/", "height": 145} colab_type="code" id="jjlhsHdF5Wtg" outputId="aca28c53-0170-4c1b-ab85-b33bf111073a"
pitcher_df17.pitch_type.value_counts()

# + colab={"base_uri": "https://localhost:8080/", "height": 126} colab_type="code" id="5ocyyIIj5oDe" outputId="8c4effe6-da81-4ac3-8303-2df850c129c8"
pitcher_df.pitch_type.value_counts()

# + colab={"base_uri": "https://localhost:8080/", "height": 90} colab_type="code" id="MwHx0Q-YB1Fj" outputId="00bbf489-9793-461a-b2e6-eb7cc1313660"
pitcher_df17.pitch_cat.value_counts(normalize=True)

# + colab={"base_uri": "https://localhost:8080/", "height": 90} colab_type="code" id="aN-xT8pFB4bi" outputId="e8c3e434-4ca3-4c60-82f6-2d95c41e016c"
pitcher_df.pitch_cat.value_counts(normalize=True)

# + colab={} colab_type="code" id="dHV8WdYSI0Pf"
pitcher_df.head(1)

# +
#pitcher_df[pitcher_df['release_speed'].isna()][['release_speed', 'pitch_type','pitch_name']]

# + [markdown] colab_type="text" id="o8tCV8zU4Wo3"
# ## Choose features to use in model input vector, re-cast datatypes, encode categorical variables (onehot and binary), scale appropriately (Standard vs MinMax vs Robust), and check for any remaining NaNs and choose imputation strategy
#
# #### Train/test Split (IMO use April/2017 - end of June/2019 as Train, and July/August 2019 as Test)

# + colab={} colab_type="code" id="sUBtQbYMJK4K"
# model_features = ['game_date', 'batter', 'on_3b', 'on_2b', 'on_1b', 'outs_when_up',
#                 'inning','fielder_2', 'sz_top', 'sz_bot', 'game_pk','at_bat_number',
#                 'pitch_number', 'bat_score', 'fld_score', 'stand', 'p_throws',
#                 'inning_topbot', 'if_fielding_alignment','score_diff', 'on_1b_id',
#                 #'L1_ball_high', 'L1_ball_low', 'L1_ball_left', 'L1_ball_right',
#                 'bases_loaded', 'fastball_perc_faced', 'fastball_chase_perc',
#                 'fastball_bip_swung_perc', 'fastball_taken_strike_perc',
#                 'fastball_est_woba', 'fastball_babip', 'fastball_iso_value',
#                 'breaking_perc_faced', 'breaking_chase_perc', 'breaking_bip_swung_perc',
#                 'breaking_taken_strike_perc', 'breaking_est_woba', 'breaking_babip',
#                 'breaking_iso_value', 'offspeed_perc_faced', 'offspeed_chase_perc',
#                 'offspeed_bip_swung_perc', 'offspeed_taken_strike_perc',
#                 'offspeed_est_woba', 'offspeed_babip', 'offspeed_iso_value',
#                 'pitchout_perc_faced', 'balls','strikes', 'of_fielding_alignment',
#                 '_count', 'count_cat', 'overall_breaking_perc',
#                 'count_cat_breaking_perc', 'overall_fastball_perc',
#                 'count_cat_fastball_perc', 'overall_offspeed_perc',
#                 'count_cat_offspeed_perc', 'pitch_count', 'L1_pitch_type',
#                 'L1_pitch_result', 'L1_pitch_zone', 'L2_pitch_type',
#                 'L2_description', 'L2_pitch_zone','L3_pitch_type',
#                 'L3_description', 'L3_pitch_zone', 'L5_fastball_perc', 
#                 'L15_fastball_perc', 'L5_breaking_perc', 'L15_breaking_perc',
#                 'L5_offspeed_perc', 'L15_offspeed_perc', 'L5_strike_perc',
#                 'L15_strike_perc', 'batting_order_slot', 'pitcher_AB', 
#                 'prev_ab_run_scored', 'prev_ab_homerun', 'prev_ab_walk',
#                 'prev_ab_basehit', 'prev_ab_strikeout', 'PB_fastball', 'PB_breaking',
#                 'PB_offspeed', 'pitch_cat']


# -

drop_cols = ['index','release_speed','release_pos_x','release_pos_z','hit_location',
             'game_year', 'pfx_x', 'pfx_z', 'plate_x', 'plate_z', 'hc_x', 'hc_y', 'vx0',
             'vy0', 'vz0', 'ax', 'ay', 'az','hit_distance_sc', 'launch_speed', 'launch_angle',
             'effective_speed', 'release_spin_rate', 'release_extension', 'release_pos_y',
             'estimated_ba_using_speedangle', 'estimated_woba_using_speedangle', 'woba_value',
             'woba_denom', 'babip_value', 'iso_value', 'launch_speed_angle', 'home_score',
             'away_score', 'fld_score', 'post_bat_score', 'post_fld_score', 'p_throws',
             'home_team', 'away_team', 'type', 'bb_type', 'sv_id', 'ball_high', 'ball_low',
             'ball_left', 'ball_right', 'batter_swung', 'in_strikezone', 'chased','events',
             'description', 'des', 'pitch_name', 'zone', 'post_away_score', 'post_home_score',
             'batter', 'on_1b_id', 'game_pk']


# + colab={} colab_type="code" id="Lu-VD-5XaVS_"
def extract_date_features(df):
    df = df.copy()
    
    #df['day'] = df['game_date'].dt.day
    df['month'] = df['game_date'].dt.month
    df['year'] = df['game_date'].dt.year
    
    df = df.sort_values(by=['game_date', 'game_pk', 'pitch_count'])
    #df = df.drop('game_date', axis=1)
    return df

#convert datetime into just month and year
model_df = extract_date_features(pitcher_df)

#drop columns before pickling
model_df = model_df.drop(columns=drop_cols)

#rename fielder_2 to catcher_id:
model_df = model_df.rename(columns={'fielder_2':'catcher_id'})

#add bool for whether there are ANY runners on base:
model_df['on_base'] = ((model_df['on_3b'] + model_df['on_2b'] + model_df['on_1b']) > 0) * 1

#fillna for on_1b_id:
#model_df['on_1b_id'] = model_df['on_1b_id'].fillna('N/A')

#fillna for infield and outfield alignment:
model_df[['if_fielding_alignment', 'of_fielding_alignment']] = model_df[['if_fielding_alignment', 'of_fielding_alignment']].fillna('Standard')

#fillna for sz_top and sz_bot
model_df['sz_top'] = model_df['sz_top'].fillna(model_df['sz_top'].median())
model_df['sz_bot'] = model_df['sz_bot'].fillna(model_df['sz_bot'].median())

#fillna for catcher_id w/ most frequent catcher for that pitcher:
catcher = model_df['catcher_id'].value_counts().index[0]
model_df['catcher_id'] = model_df['catcher_id'].fillna(catcher)

#convert cols to categorical:
cat_cols = ['pitch_type', 'balls', 'strikes', 'player_name', 'of_fielding_alignment', '_count', 'count_cat', 'pitch_cat',
              'L1_pitch_type', 'L1_pitch_result', 'L1_description', 'L1_pitch_zone', 'L1_batter_swung', 'L1_chased', 'L2_pitch_type',
              'L2_pitch_result', 'L2_description', 'L2_pitch_zone', 'L2_chased', 'L3_pitch_type', 'L3_description', 'L3_pitch_zone',
              'L3_chased', 'batting_order_slot', 'pitcher', 'outs_when_up', 'inning', 'catcher_id', 'year', 'month',
              'at_bat_number', 'pitch_number', 'pitch_count']
model_df[cat_cols] = model_df[cat_cols].astype('category')


# + colab={"base_uri": "https://localhost:8080/", "height": 98} colab_type="code" id="ojxoymhk9YIl" outputId="d13f5dad-a13d-4921-b520-90bf7a2413e8"
model_df.head()

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} colab_type="code" id="CETjjIB23Z7d" outputId="908548d7-1c43-4c7a-a7eb-ac92e0158d80"
#model_df.isna().sum()

# +
#model_df.info()

# + colab={} colab_type="code" id="SUGj-PZM3RQ_"
# import category_encoders as ce
# from sklearn.preprocessing import RobustScaler, StandardScaler

# #categoricals cols:


# #one hot encode categorical columns:
# one_hot_cols = ['outs_when_up', 'stand', 'inning_topbot', 'if_fielding_alignment',
#                 'of_fielding_alignment', 'count_cat', 'L1_pitch_type', 
#                 'L1_pitch_result', 'L1_ball_high', 'L1_ball_low', 'L1_ball_left', 
#                 'L1_ball_right','year', 'p_throws', 'L2_pitch_type',
#                 'L2_description', 'L2_pitch_zone','L3_pitch_type',
#                 'L3_description', 'L3_pitch_zone']

# #binary encode categorical columns:
# binary_cols = ['inning', 'batter', 'catcher_id', 'game_pk', 'at_bat_number', 'pitch_number',
#                'bat_score', 'fld_score', 'score_diff', 'on_1b_id', 'balls', 'strikes', 
#                '_count', 'pitch_count', 'L1_pitch_zone', 'batting_order_slot', 
#                'month']
    

# #robust scaler cols:

# numeric_cols = ['fastball_perc_faced', 'fastball_chase_perc', 'fastball_bip_swung_perc', 
#                 'fastball_taken_strike_perc', 'fastball_est_woba', 'fastball_babip', 
#                 'fastball_iso_value', 'breaking_perc_faced', 'breaking_chase_perc', 
#                 'breaking_bip_swung_perc', 'breaking_taken_strike_perc', 
#                 'breaking_est_woba', 'breaking_babip', 'breaking_iso_value', 
#                 'offspeed_perc_faced', 'offspeed_chase_perc', 'offspeed_bip_swung_perc', 
#                 'offspeed_taken_strike_perc','offspeed_est_woba', 'offspeed_babip', 
#                 'offspeed_iso_value','overall_breaking_perc', 'count_cat_breaking_perc', 
#                 'overall_fastball_perc', 'count_cat_fastball_perc', 'overall_offspeed_perc', 
#                 'count_cat_offspeed_perc', 'pitchout_perc_faced', 'L5_fastball_perc', 
#                 'L15_fastball_perc', 'L5_breaking_perc', 'L15_breaking_perc',
#                 'L5_offspeed_perc', 'L15_offspeed_perc', 'L5_strike_perc', 
#                 'L15_strike_perc', 'PB_fastball', 'PB_breaking', 'PB_offspeed', 
#                 'sz_bot', 'sz_top']

# def final_model_preprocess(model_df):
#     df = model_df.copy()
    
#     #one hot encode certain columns
#     one_hot_ce = ce.OneHotEncoder(cols=one_hot_cols, drop_invariant=False, 
#                                   return_df=True, verbose=10, use_cat_names=True)
    
#     one_hot_df = df[one_hot_cols].copy() 
#     one_hot_df = one_hot_ce.fit_transform(one_hot_df)
    
#     #binary encode columns w/ lots of categories
#     binary_ce = ce.BinaryEncoder(cols=binary_cols, return_df=True)
    
#     binary_df = df[binary_cols].copy()
#     binary_df = binary_ce.fit_transform(binary_df)
    
#     #grab the remaining features
#     other_cols = set(df.columns) - set(one_hot_cols + binary_cols)
#     other_df = df[other_cols].copy()
    
#     #merge everything back together
#     df = pd.concat([one_hot_df, binary_df, other_df], axis=1)
#     return df

# model_df = final_model_preprocess(model_df)

# + colab={"base_uri": "https://localhost:8080/", "height": 35} colab_type="code" id="gsRafjtAMgMl" outputId="c53aa73a-b217-4043-c805-ee8f68cbec6a"
model_df.shape

# + colab={} colab_type="code" id="DoRFIc74lz8v"
#Todo:

# - pickle the model_df and export

# - scale the data (probably w/ robust scaler)

# - split into train and test

# - for non tree based models: encode categorical variables

# - train models and evaluate predictions

# + colab={} colab_type="code" id="mAnpQ4c8oY7R"
vals = list(pitcher_id_dict.keys())
keys = list(pitcher_id_dict.values())
reverse_dict = dict(zip(keys, vals))

pickle_path = '/home/ec2-user/SageMaker/RC-v1.2-Predictive-Modelling/pitcher_df_pickles/'
pitcher_name = reverse_dict[pitcher_id].split()[1]

model_df.to_pickle(path=(pickle_path + pitcher_name + '_df.pkl'),compression='zip')
# -







StarcoderdataJupyterScriptsDedupFiltered