# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ### Data Exploration for Foundations in Data Science Team Project
# Start by reading in the pickle file from the data load
#

sx ls ../data

# +
import pickle
import os
import glob
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn
from sklearn.decomposition import PCA

df = pd.read_pickle('../data/cleandf.pkl')
# -

df.head(4)

new_df = df.filter(regex='zinc2_x|hud_med')
new_df.head(4)
new_df.describe()

# +
corr_new_df = new_df.corr(method='pearson')
print('----------Correlations using Pearson Method----------')
print(corr_new_df)
print('----------Heatmap based on above correlations----------')
mask = np.zeros_like(corr_new_df)
mask[np.triu_indices_from(mask)] = True
seaborn.heatmap(corr_new_df, cmap='RdYlGn_r', vmax=1.0, vmin=-1.0, mask = mask, linewidths=2.5)

plt.yticks(rotation=0) 
plt.xticks(rotation=90) 
plt.show()
# -

# investigate own_rent
df.own_rent_x.unique()

# Seperate out the features and set OWN_RENT_X as the target
df_y = df['own_rent_y']


features = ['assisted_y', 'structure_type_y', 'n_units_y', 'vacancy_y', 'cost12_rel_ami_cat_y', 'rooms_y', 'bedrms_y', 'value_y', 'cost08_rel_ami_cat_y', 'utility_y', 'cost12_rel_fmr_cat_y', 'abl_80_y', 'cost_12', 'fmr_y', 'cost12_rel_ami_pct_y', 'cost06_rel_ami_cat_y', 'cost12_rel_fmr_pct_y', 'cost12_rel_pov_pct_y', 'age1_y','cost_08_y', 'cost12_rel_pov_cat_y', 'inc_pel_ami_cat','abl_50_y','other_cost_y','cost08_rel_fmr_cat_y']
df_x = df.loc[:, features].values
df_x


df_features = pd.DataFrame(df_x)
df_next = df_features.replace([np.inf, -np.inf], np.nan)
#df_features.head(1000)
#df_cleaned = df_features.dropna(how='any') 

df_next.describe()
df_cleaned = df_next.dropna(how='any') 
df_cleaned.isna().all()
df_cleaned = np.ndarray(df_cleaned)

np.nan_to_num(df_x)


from sklearn.preprocessing import StandardScaler
df_cleaned = StandardScaler().fit_transform(df_x)

from sklearn.decomposition import PCA
pca = PCA(n_components=2)
principalComponents = pca.fit_transform(df_cleaned)


# +
principalDf = pd.DataFrame(data = principalComponents
             , columns = ['principal component 1', 'principal component 2'])


# -

finalDf = pd.concat([principalDf, df[['own_rent_y']]], axis = 1)


# +
fig = plt.figure(figsize = (8,8))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)

ax.set_title('2 component PCA', fontsize = 20)
targets = ['Own', 'Rent']
colors = ['r', 'g']
for target, color in zip(targets,colors):
    indicesToKeep = finalDf['own_rent_y'] == target
    ax.scatter(finalDf.loc[indicesToKeep, 'principal component 1']
               , finalDf.loc[indicesToKeep, 'principal component 2']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()
# -



StarcoderdataJupyterScriptsDedupFiltered