# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/shrey-cloud/CustomSpeechRecognizer/blob/master/PCA.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="qOD71vDmq6dm"
# ##PCA
#

# + colab={"base_uri": "https://localhost:8080/", "height": 206} id="ytf7HUmkxkK9" outputId="c5a72b31-1da0-41b6-fd7e-d3fcbdd8957e"
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
# %matplotlib inline
column_names = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV']
dataset = pd.read_csv('https://raw.githubusercontent.com/niranjangirhe/dataset/main/housing.csv', header=None, delimiter=r"\s+", names=column_names)

dataset.head()

# + colab={"base_uri": "https://localhost:8080/"} id="O3-RyjfAxZpv" outputId="d73ccec0-a389-421b-933e-9507f8dc8251"
print(dataset)


# + id="7SE2WOHH0JVu"
class PCA:
    
    # I will denote components as features here
    # Though its not mathematically accurate but machine learning it works 
    def __init__(self,number_of_important_features=2):
        # number of specified features
        # Default being passed as 2
        self.number_of_important_features=number_of_important_features
        # Best possible features
        self.features=None
        self._mean=None
        
        
    def fit(self,X):
        # placing mean to as origin of axis
        # axis =0 is mean of rows along the column direction 
        self._mean=np.mean(X,axis=0)
        X=X-self._mean
        
        # Co-variance of N,D -->DxD
        # Also called Autocorrelation as both are X's
        covariance=np.dot(X.T,X)/(X.shape[0]-1)
        print(covariance.shape)
        
        # Eigenvalues,eigenvectors detail discussion below
        # Eigenvector is the vector which doesnot chnage it span(simply, direction) after matrix transformation
        # So, why eigen importance. Best intuitive way to say
        # for 3D object, the eigenvector represents its axis of rotation(For earth eigenvector is the axis of rotation)
        # Formula A(matrix).v(eigenvector)=lambda(eigenvalue).v(eigenvector)
        # So, Intuitively above formula means, matrix transformation of eigenvector is the eigenvector scaled by eigenvalue
        # Here we are finding the eigenvector and eigenvalue of the covariance matrix
        # how to solve is (A-lambda.I(identity matrix))-v=0,  As v is non-zero --> det(A-lambda.I)=0(area under transformation=0)
        # Here lambda is the knob by tweaking it, we change the det = 0
        # We can do all this by only one line of code, isnt it awesome!!!
        # There is very powerful application of eigen's i.e eigenbasis-->diagonalisation()
        # A gift for the patience
        # you can say this to your gf or bf --> "My love for you is like eigenvector"
        eigenvalues,eigenvector=np.linalg.eig(covariance)
        print("eigenvalues-->",eigenvalues.shape)
        print("eigenvalues \n",eigenvalues)
        print("eigenvector-->",eigenvector.shape)
        print("eigenvector \n",eigenvector)
        #sort the eigenvalues from highest to lowest
        # If we didnt transpose, then applying indexs will require more steps and computation
        eigenvector=eigenvector.T
        print("eigenvector.T-->",eigenvector.shape)
        print("eigenvector after Transpose\n",eigenvector)
        indexs=np.argsort(eigenvalues)[::-1]
        #taking those indices and storing in eigenvalues and eigenvectors accordingly
        eigenvector=eigenvector[indexs]
        print("eigenvector-indexs-->",eigenvector.shape)
        print("eigenvector after indexes \n",eigenvector)
        eigenvalues=eigenvalues[indexs]
        print("eigenvalues-indexs-->",eigenvalues.shape)
        print("eigenvalues \n",eigenvalues)
        
        ## This below code snippet is for seeing how to determine which feature to be calculated
        total = sum(eigenvalues)
        variance_of_each_feature = [(i / total)*100 for i in eigenvalues]
        print("variance of each feature-->",variance_of_each_feature)
        
        # Now taking only number of specified componenets
        self.features=eigenvector[:self.number_of_important_features]
        print("self.features",self.features.shape)
        # So, now the we have chosen most significant features componenet
        
    def apply(self,X):
      # Here we project the data onto Principal component line
        X=X-self._mean
        # Check the dimensionality with (.shape) to confirm for yourselves
        # Here X-->(N,4);self.features-->2,4
        # (X,self.features.T)-->(N,4)x(4,2)==(N,2) i.e N samples with 2 feature vector 
        return np.dot(X,self.features.T)

# + id="gXvkSM710uJE"
X = dataset.iloc[:,:]

# + colab={"base_uri": "https://localhost:8080/"} id="hEwzPvuw0xij" outputId="31bea069-5f87-4410-a331-82cb58fc8088"
print(X)

# + colab={"base_uri": "https://localhost:8080/"} id="oL5bx5vH0e5n" outputId="dbd27b41-af5c-447e-e359-1a2f45d002fb"
from sklearn.preprocessing import StandardScaler
X = StandardScaler().fit_transform(X)
print(X[0:5])

# + id="5xvThC3z2JQF"
y = dataset['MEDV']

# + id="TNUZZ0NRknk_" colab={"base_uri": "https://localhost:8080/"} outputId="a604d52d-9cba-43c1-9762-608edc02415e"
from sklearn.feature_selection import SequentialFeatureSelector as sfs
#I am going to use RandomForestRegressor algoritham as an estimator. Your can select other regression alogritham as well.
from sklearn.ensemble import RandomForestRegressor
#k_features=10 (It will get top 10 features best suited for prediction)
#forward=True (Forward feature selection model)
#verbose=2 (It will show details output as shown below.)
#cv=5 (Kfold cross valiation: it will split the training set in 5 set and 4 will be using for training the model and 1 will using as validation)
#n_jobs=-1 (Number of cores it will use for execution.-1 means it will use all the cores of CPU for execution.)
#scoring='r2'(R-squared is a statistical measure of how close the data are to the fitted regression line)
model_forward=sfs(RandomForestRegressor(),n_features_to_select=7, direction = 'forward')
model_forward.fit(X,y)
model_backward=sfs(RandomForestRegressor(),n_features_to_select=7, direction = 'backward')
model_backward.fit(X,y)

# + colab={"base_uri": "https://localhost:8080/"} id="LhyDu2nLncJy" outputId="5c486efd-2da7-48d1-bf1e-6ff01ab813c1"
Wrapper_support_forward = model_forward.get_support()
Wrapper_support_backward = model_backward.get_support()
print(Wrapper_support_forward)
print(Wrapper_support_backward)
print(column_names)

# + colab={"base_uri": "https://localhost:8080/"} id="Yu-zzLhBn1RE" outputId="fcf6316f-75ae-495b-dde0-9b2205258d7a"
print("This are the selected features using Wrapper with Random FOrest and forward approach")
for i in range(14):
  if(Wrapper_support_forward[i] == True):
      print(column_names[i])
print("This are the selected features using Wrapper with Random FOrest and backward approach")
for i in range(14):
  if(Wrapper_support_backward[i] == True):
      print(column_names[i])

# + [markdown] id="GNLKb1XSkiAx"
# Embeded with Decision Tree

# + id="YtgHK3jCgzQT"
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
from sklearn.preprocessing import MinMaxScaler
X_norm = MinMaxScaler().fit_transform(X)


# + colab={"base_uri": "https://localhost:8080/"} id="ZoobNc_ecOHi" outputId="c45353af-49b8-4baa-aa8c-f61b250551a2"
from sklearn.feature_selection import SelectFromModel
from sklearn.tree import DecisionTreeRegressor

embeded_dt_selector = SelectFromModel(DecisionTreeRegressor(), threshold='1.25*median')
embeded_dt_selector.fit(X_norm, y)

# + colab={"base_uri": "https://localhost:8080/"} id="iAKs-hyecYAp" outputId="7fe5c9fe-021a-4ceb-db27-6be7a31e369f"
embeded_dt_support = embeded_dt_selector.get_support()
print(embeded_dt_support)
print(column_names)
#embeded_rf_feature = X.iloc[:,embeded_rf_support].columns.tolist()
#print(str(len(embeded_rf_feature)), 'selected features')

# + colab={"base_uri": "https://localhost:8080/"} id="vs2WD6HYj8V2" outputId="7ff3b412-9fee-4090-8160-4c26a608cd76"
print("This are the selected features using Embeded with Decisiomn tree")
for i in range(14):
  if(embeded_dt_support[i] == True):
      print(column_names[i])
  

# + [markdown] id="-fGMbNAWkf83"
# PCA

# + id="DmIFrewC06GI"
pca=PCA(2)

# + colab={"base_uri": "https://localhost:8080/"} id="Hn_fjAa_0_34" outputId="9c04aa9e-3eff-49d8-c6c0-cf3951e8278d"
pca.fit(X)

# + id="qMXAYc2B179s"
projected=pca.apply(X)

# + id="kBY0pTNE2Azu"
x0=projected[:,0]
x1=projected[:,1]

# + colab={"base_uri": "https://localhost:8080/", "height": 284} id="QbgOZjCn2VjL" outputId="fd19649f-11d7-485b-ccf3-1476c74bda81"
plt.scatter(x0,x1,c=y)

# + id="vF4W38KbDPbi"


StarcoderdataJupyterScriptsDedupFiltered