import pandas as pd
%matplotlib inline

"""
Explanation: An overview of feature engineering for regression and machine learning algorithms
End of explanation
"""


df = pd.DataFrame({'key':['b','b','a','c','a','b'],'data1':range(6)})

df

pd.get_dummies(df['key'],prefix='key')

"""
Explanation: A simple example to illustrate the intuition behind dummy variables
End of explanation
"""


df = pd.read_csv('data/ontime_reports_may_2015_ny.csv')

#count number of NaNs in column
df['ARR_DELAY'].isnull().sum()

#calculate the percentage this represents of the total number of instances
df['ARR_DELAY'].isnull().sum()/df['ARR_DELAY'].sum()

"""
Explanation: Now we have a matrix of values based on the presence of absence of the attribute value in our dataset
Another example with the flight statistics
Now let's look at another example using our flight data
End of explanation
"""


#filter ARR_DELAY NaNs
df = df[pd.notnull(df['ARR_DELAY'])]

"""
Explanation: We could explore whether the NaNs are actually zero delays, but we'll just filter them out for now, especially since they represent such a small number of instances
End of explanation
"""


#code whether delay or not delayed
df['IS_DELAYED'] = df['ARR_DELAY'].apply(lambda x: 1 if x>0 else 0 )

#Let's check that our column was created properly
df[['ARR_DELAY','IS_DELAYED']]

pd.get_dummies(df['ORIGIN'],prefix='origin') #We'd want to drop one of these before we actually used this in our algorithm

"""
Explanation: We can discretize the continuous DEP_DELAY value by giving it a value of 0 if it's delayed and a 1 if it's not. We record this value into a separate column. (We could also code -1 for early, 0 for ontime, and 1 for late)
End of explanation
"""


df = pd.read_csv('data/heights_weights_genders.csv')

pd.get_dummies(df['Gender'],prefix='gender').corr()

"""
Explanation: Multicollinearity
End of explanation
"""


from sklearn import preprocessing

x = df[['Height','Weight']].values #returns a numpy array
min_max_scaler = preprocessing.MinMaxScaler()
x_scaled = min_max_scaler.fit_transform(x)
df_normalized = pd.DataFrame(x_scaled)

df['Height'].describe()

df_normalized

"""
Explanation: <img src="http://i.giphy.com/3ornka9rAaKRA2Rkac.gif"></img>
That's the dummy variable trap
We can also normalize variables across a range
End of explanation
"""


