"""
# **Principal Component Analysis**

> *Reducing dimensions by*
>
> - **Maximizing variance** of projections on *optimal component* (or)
> - **Minimizing distance** between an *optimal component* and original data samples*
"""
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

np.random.seed(123)
"""
## **Background**
"""
"""
> **High variance $\Rightarrow$ High information**
>
> While reducing dimensions, make sure **information is lost as less as possible i.e variance is as high as possible**
>
> **Intuition:**<br>
> Imagine we have car dataset with 2 features `colors` and `num_of_wheels`. Which of the two featues can help us differentiate between *different cars*? `colors` because of **high variability (has large amount of info)**. Because of **zero variability(less/no info)**, we cannot use `num_of_wheels`
"""
def minmaxscale(x):
    return (x-x.min()) / (x.max()-x.min())

# plot 1
x1 = np.random.uniform(4, 5, 50)
x2 = np.random.uniform(0, 10, 50)

# plot 2
errors = np.random.uniform(0, 2, 50)
x3 = np.random.uniform(5, 3, 50)
x4 = 3*x3 + 3 + errors

# subplots
fig, axarr = plt.subplots(1,2)
fig.set_size_inches(10,4)

axarr[0].scatter(x1, x2)
axarr[0].arrow(4.5, 2, 0, 6, color="red", width=0.10)
arrow = axarr[0].arrow(4.5, 6, 0, -4, color="red", width=0.10)

axarr[0].title.set_text("Consider reducing 2-D data to 1-D")
axarr[0].set_xlabel("feat1 (on x-component of cartesian plane)")
axarr[0].set_ylabel("feat2 (on y-component of cartesian plane)")
axarr[0].legend(["original data"])
axarr[0].legend([arrow,], ['Optimal component',])
axarr[0].set_xticks(range(0, 10))
axarr[0].set_yticks(range(0, 10))

axarr[1].scatter(x3, x4)
arrow = axarr[1].arrow(3, 13, 1.6, 5, color="red", width=0.10)
axarr[1].arrow(4.6, 18, -1.6, -5, color="red", width=0.10)

axarr[1].title.set_text("Consider reducing 2-D data to 1-D")
axarr[1].set_xlabel("feat3 (on x-component of cartesian plane)")
axarr[1].set_ylabel("fea4 (on y-component of cartesian plane)")
axarr[1].legend(["original data", "optimal component"])
axarr[1].legend([arrow,], ['Optimal component',])
axarr[1].set_xticks(range(0, 10))
axarr[1].set_yticks(range(10, 20))

plt.show()
"""
The projections(shadows) of original data points on red-lines(components) will be our new data whose dimension has been reduced for 2-D to 1-D

> Because, these **projected** points will have maximum variance. Thus, information is retained
"""
"""
### Projections

Projection of vector a on vector b,<br>
$$\text{Vector:}\,\, \vec{p_{a,b}} = \frac{a \cdot b}{{||b||}^{2}} \hat{b}$$
$$\text{Scalar:}\,\, |p_{a,b}| = \frac{a \cdot b}{{||b||^2}} $$

Note that projections **conserve mean** i.e *projection of mean will be a mean*. And $||x||^2 = x \cdot x = (x^T x)$
"""
"""
# **Procedure:**

1. Colum standardize data into ${}^{standardized}{X_{nxm}}$
2. Find optimal components (col vectors with size (mx1)) $u_1, u_2, u_3 \dots u_d$ (d<m)
3. Generate ${X_{nxd}}^{dim\, reduced}$ w/ help of projections

$${}^{dim\, reduced}{X_{n\text{x}d}} = {}^{standardized}{X_{n\text{x}m}} \cdot {U_{m\text{x}d}}$$ 
"""
"""
## **Finding Optimal Components**

**OBJECTIVE:** To find *direction $\hat{u_1}$* (optimal component) 

This is an optimisation problem. It can be solve in two ways

- Find *direction $\hat{u_1}$* such that **variance of projected points on $\hat{u_1}$ is MAXIMIZED**
- Find *direction $\hat{u_1}$* such that **distance b/w original datapoints and $\hat{u_1}$ is MINIMIZED**
"""
"""
## Method 1of2: 

> Find direction  𝑢̂ (comonent) such that variance of projected points on  𝑢̂   is MAXIMIZED
"""
"""
**Optimisation Problem:**
<span style="color:blue">
$$Maximize_{u_1}\,\, \Bigg\{ \text{var(projection of original-data on u1)} \Bigg\} \,\,\, \text{such that } u_1=\hat{u_1}\,\, \text{i.e } {||u_1||}^{2} = 1 (constant)$$
</span>
    
$$
\Rightarrow Max_{u_1}\,\, {{\Bigg\{ var {\bigg( \frac{x_i \cdot u_1}{||u_1||^2} \bigg)}^{i=1,2\dots n}} \Bigg\}} \,\,\, \text{where, } x_i \in \text{X - High dim original data and n - num of samples} \\
\Rightarrow Max_{u_1}\,\, {\Bigg\{ var {\bigg( x_i \cdot u_1 \bigg)}^{i=1,2\dots n}} \Bigg\}\,\,\, \because \text{||u||=1} \\
\Rightarrow Max_{u_1}\,\, \Bigg\{   \frac{1}{n}\sum{({u_1}^T x_i - {u_1}^T \cdot \mu)^2}    \Bigg\}
\Rightarrow Max_{u_1}\,\, \Bigg\{   \frac{1}{n}\sum{({u_1}^T x_i)^2}    \Bigg\} \,\,\, \because \text{col-standardized(mean=0)}
$$ 


Hence, the optimisation problem with constraints is,

<span style="color:green">
$$ Max_{u_1}\,\, \Bigg\{   \frac{1}{n}\sum_{i=1}^{n}{({u_1}^T x_i)^2}    \Bigg\} \,\,\, \text{such that } ||u_1||^2 = 1 $$
</span>


> $||u_1||^2 = 1$ because, it must be constant. Otherwise, we can simply put $u_1 = \infty$ for maxizing variance
"""
"""
## Method 2of2: 

> Find direction  𝑢̂ (component) such that distance b/w original datapoints and $\hat{u_1}$ is MINIMIZED
"""
"""
Right angled triangle is formed by --
- distance $d_i$ b/w original data-point $x_i$ and it's projected point on $\vec{u_1}$ 
- magnitude of original datapoint vector i.e $||\vec{x_1}||^2$ 
- magnitude of projection $||\vec{p}||^2$
"""
fig = plt.figure(figsize=(7,4))

# cartesian plane start------------------------------------------
begx, begy = 0,0
dx, dy = 4,0
plt.arrow(begx, begy, dx, dy, color="black", width=0.001, head_width=0.04, head_length=0.04)

begx, begy = 0,0
dx, dy = -4,0
plt.arrow(begx, begy, dx, dy, color="black", width=0.001, head_width=0.04, head_length=0.04)

begx, begy = 0,0
dx, dy = 0,3
plt.arrow(begx, begy, dx, dy, color="black", width=0.001, head_width=0.04, head_length=0.04)

begx, begy = 0,0
dx, dy = 0,-4
plt.arrow(begx, begy, dx, dy, color="black", width=0.001, head_width=0.04, head_length=0.04)
# cartesian plane end------------------------------------------


# optimal component
begx, begy = 0,0
endx, endy = 5,3
plt.plot([begx, endx], [begy, endy],linestyle="-", color="red", label="optimal component u_1")

# points
plt.scatter([2.5], [2.5], color="blue", label="original datapoint x_i")
plt.scatter([3], [1.8], color="green", label="projection of x_i")

# distance d
begx, begy = 2.5, 2.5
endx, endy = 3, 1.8
plt.plot([begx, endx], [begy, endy],linestyle="--", color="orange", label="distance d_i")

# original point vector
begx, begy = 0, 0
endx, endy = 2.5, 2.5
plt.plot([begx, endx], [begy, endy],linestyle="-", color="blue", label="x_i datapoint vector magnitude")

# magnitude of projection
begx, begy = 0, 0
endx, endy = 3, 1.8
plt.plot([begx, endx], [begy, endy],linestyle="--", color="green", label="x_i projection magnitude")

plt.title("Right-angled Triangle")
plt.axis('off')
plt.legend()
plt.show()
"""
$$\text{By pythagoras theorem,}\,\,\, d^2 = ||\vec{x_i}||^2 - p^2 = \bigg( (x_i^T x_i) \,-\, (u_1^T x_i)^2 \bigg) \,\, \because ||u_1||=1$$
"""
"""
**Optimisation Problem:**
<span style="color:blue">
$$Minimize_{u_1}\,\, \Bigg\{ \text{dist_metric(between x_i and it's projection)} \Bigg\} \,\,\, \text{such that } u_1=\hat{u_1}\,\, \text{i.e } {||u_1||}^{2} = 1 (constant)$$
</span>

Here, dist_metric is euclidean distance $d^2$ <br>
Thus the optimisation problem with constraints is,

<span style="color:green">
$$ Min_{u_1} \Bigg\{  (x_i^T x_i) \,-\, (u_1^T x_i)^2  \Bigg\} \,\,\, \text{such that } ||u_1||^2 = 1 $$
</span>


> <br>
> <b>Note that both optimisation problems look almost similar and can be proved that they are same [proof]()</b><br>
> 
> Solving the above two optimisation problems are same as finding **Eigen vectors**
"""
"""
## Eigen Vectors

> *Linear algebra technique. Interpreted differently in differenet areas of study* [(know more)](https://www.math.ubc.ca/~pwalls/math-python/linear-algebra/eigenvalues-eigenvectors/)

$$ S \vec{v} = \lambda \vec{v} \,\,\,\, \text{where S is covariance matrix (symetric square) and } \lambda \text{ is a scalar quantity}$$

Solving the above equation gives different $\lambda$ values called **EIGEN VALUES** such that,

$$\lambda_1 \ge \lambda_2 \ge \lambda_3 \ge \lambda_4 \ge \dots \lambda_d$$

Each $\lambda$ value has an **EIGEN VECTOR** $\vec{v}_k$ assosciated with it. $\vec{v}_k$ is column vector of size $\text{(1 x d)}$ where d is size of $S$

<br>

---
**Significance of $\vec{v}$ values**
> All are mutually perpendicular

**Significance of $\lambda$ values**

> $$\text{var % covered by eigen } \vec{v}_k \text{ compared to other GENERATED eigenvetors under consideration} = \frac{\lambda_k}{\sum_{i=1}^d{\lambda_i}}$$
>
> Used to **decide the num of dimensions** given, the amount of variability we want to retain
> 
> For example, if **`x`% variability must be retained**, find minimum dims `k` such that **$\sum^k_{i=1}{\lambda_i} \ge x $**
"""
fig, axarr = plt.subplots(1,4)
fig.set_size_inches(20,4)

# 1. perfect slope
x1 = np.random.uniform(3,4,100)
x2 = 3*x1 + 4

axarr[0].set_title("Fig1: Perfect slope\nλ2 = 0 ")
axarr[0].scatter(x1, x2)
axarr[0].plot([3.2, 3.88], [15.75,13], color="red")
axarr[0].plot(x1, x2, color="red")
axarr[0].grid()
axarr[0].set_xlabel("feat1")
axarr[0].set_ylabel("feat2")


# 2. Fig2:
_x1 = np.random.uniform(3,4,100)
errors = np.random.normal(0,0.13,100)
_x2 = 3*_x1 + 4 + errors

axarr[1].set_title("Fig2: λ1>>λ2")
axarr[1].scatter(_x1, _x2)
axarr[1].plot(x1, x2, color="red")
axarr[1].plot([3.2, 3.83], [15.75,13], color="red")
axarr[1].grid()
axarr[1].set_xlabel("feat1")
axarr[1].set_ylabel("feat2")


# 3. Fig3:
_x1 = np.random.uniform(3,4,100)
errors = np.random.normal(0,0.6,100)
_x2 = 3*_x1 + 4 + errors

axarr[2].set_title("Fig3: λ1>λ2")
axarr[2].scatter(_x1, _x2)
axarr[2].plot(x1, x2, color="red")
axarr[2].plot([3.3, 3.66], [17,11], color="red")
axarr[2].grid()
axarr[2].set_xlabel("feat1")
axarr[2].set_ylabel("feat2")


# 4. Fig4: Perfect circle
from sklearn.datasets import make_blobs
X, y = make_blobs(n_samples=1000, centers=1, n_features=2, random_state=0)

axarr[3].set_title("Fig4: Perfect circle\nλ1=λ2")
axarr[3].scatter(X.T[0], X.T[1])
axarr[3].plot([-1,3],[2,7], color="red")
axarr[3].plot([-2,4],[7,2], color="red")
axarr[3].grid()
axarr[3].set_xlabel("feat1")
axarr[3].set_ylabel("feat2")

plt.show()
"""
Consider the above case, converting **2-D data to new 2-D data (instead of dim reduction)**,

The two new components will be $\hat{v_1}$ and $\hat{v_2}$ corresponding to $\lambda_1$ and $\lambda_2$ (found using above formula)
> *But what does $\lambda_1$ and $\lambda_2$ mean?*

- $\lambda_1$ $\ge \lambda_2$ Does **NOT** imply $\text{variance-of-projections-of}\, \hat{v_1}$ $\ge \text{variance-of-projections-of}\, \hat{v_2}$ (Cannot be used for comparison)
- $\frac{\lambda_1}{\sum_{i=1}^d{\lambda_i}}$ $\ge \frac{\lambda_2}{\sum_{i=1}^d{\lambda_i}} \Rightarrow \text{variance-of-projections-of}\, \hat{v_1}$ $\ge \text{variance-of-projections-of}\, \hat{v_2}$ (Can be used for comparison)

> Note: we are [normalizing by division by sum](https://www.kaggle.com/l0new0lf/02-08-normalisation-vs-standardisation-vs-probs)
"""
"""
**PONDER**

1. Use of NN to find $v_1$ in $x_i^{new} = v_1^T x_i$ (simple back propagation w/ metric -- maximize variance of $x_i^{new}$)
2. As all $v_1$, $v_2$, $ ... $ are mutually perpendicular, wouldn't it be sufficient to just find one of them and then use slope formula for perpendicular lines to find others
"""
"""
# Limitations

![image.png](attachment:image.png)
"""
"""
# **Implementation**

- For visualisation
- For dimension reduction
"""
"""
## 01. Visualisation
"""
# REPRODUCABLE CODE: Visualisation only
# =====================================

#libraries
import numpy as np
import pandas as pd
from scipy.linalg import eigh 
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
import seaborn as sns

def get_pca_df(X, labels):
    
    # 1.standardize
    # ==============
    standardized_data = StandardScaler().fit_transform(X)
    covar_matrix = np.matmul(standardized_data.T , standardized_data)

    # original dims of original data (m)
    m = len(covar_matrix)
    
    # 2.Get first-two eigen vectors
    # =============================
    values, vectors = eigh(covar_matrix, eigvals=(m-2, m-1)) #(62, 63) for MNIST_8X8 i.e from end to front
    U = vectors.T # shape (d, m) where

    #print(values)    

    # 3. Genereate new 2-D data (projections)
    # ======================================
    projections = np.matmul(U, standardized_data.T) #(d,m) x (m, n) => (d, n)  [where, n - num_of_samples ]
    # appending label to the new 2-D data 
    projections = np.vstack((projections, labels)).T 

    # creating a new data frame for ploting the labeled points.
    dataframe = pd.DataFrame(data=projections, columns=("1st_principal", "2nd_principal", "label"))
    
    return dataframe

""" # Plotting
# plot bc 
sns.FacetGrid(df2, hue="label", height=6).map(plt.scatter, '1st_principal', '2nd_principal').add_legend()

plt.title("Breast Cancer Dataset \nm={} n={}".format(X_bc.shape[1], X_bc.shape[0]))
plt.show()
"""
"""
### MNIST Dataset
"""
from sklearn.datasets import load_digits
digits = load_digits()
X_mnist = digits.data
labels_mnist = digits.target
df = get_pca_df(X_mnist, labels_mnist)
# X - all numerical rvs
# label - categorical rv
# plot
sns.FacetGrid(df, hue="label", height=6).map(plt.scatter, '1st_principal', '2nd_principal').add_legend()
plt.title("MNIST Dataset \nm={} n={}".format(X_mnist.shape[1], X_mnist.shape[0]))
plt.show()
"""
## Breast Cancer Dataset
"""
# load bc data
from sklearn.datasets import load_breast_cancer
bc = load_breast_cancer()
X_bc = bc.data
labels_bc = bc.target
df2 = get_pca_df(X_bc, labels_bc)
# X - all numerical rvs
# label - categorical rv
# plot bc 
sns.FacetGrid(df2, hue="label", height=6).map(plt.scatter, '1st_principal', '2nd_principal').add_legend()

plt.title("Breast Cancer Dataset \nm={} n={}".format(X_bc.shape[1], X_bc.shape[0]))
plt.show()
"""
> 
"""
"""
## 02. Analytical dimensionality reduction
"""
# REPRODUCIBLE CODE:
# PCA for dimensionality redcution (non-visualization)
# ====================================================
import matplotlib.pyplot as plt
import numpy as np
from sklearn import decomposition
pca = decomposition.PCA()

def pca_anaysis(sample_data):
    """Sample data is `X` with all numerical rvs"""
    # connfig and transform
    pca.n_components = sample_data.shape[1]
    pca_data = pca.fit_transform(sample_data)

    # calculate cumulative variance
    percentage_var_explained = pca.explained_variance_ / np.sum(pca.explained_variance_);
    cum_var_explained = np.cumsum(percentage_var_explained)

    # Plot the PCA spectrum
    plt.figure(1, figsize=(6, 4))

    plt.clf()
    plt.plot(cum_var_explained, linewidth=2)
    plt.axis('tight')
    plt.grid()
    plt.xlabel('n_components')
    plt.ylabel('Cumulative_explained_variance')
    plt.title("PCA Analysis")
    plt.show()
    
import pandas as pd
def pca_get_df(sample_data, labels, new_dims=2):
    """Sample data is `X` with all numerical rvs"""
    pca.n_components = new_dims
    pca_data = pca.fit_transform(sample_data)
    
    df = pd.DataFrame(pca_data)
    df['labels'] = labels
    return df
"""
### MNIST Data
"""
from sklearn.datasets import load_digits
digits = load_digits()
X_mnist = digits.data
labels_mnist = digits.target
pca_anaysis(X_mnist)
"""
**Observation**
> If we take 20-dimensions, approx. 90% of variance is expalined.
"""
pca_get_df(X_mnist, labels_mnist, new_dims=20).head()
df = pca_get_df(X_mnist, labels_mnist, new_dims=2)
# plot bc 
import seaborn as sns
sns.FacetGrid(df, hue="labels", height=6).map(plt.scatter, 0, 1).add_legend()

plt.title("Mnist Dataset \nm={} n={}".format(X_mnist.shape[1], X_mnist.shape[0]))
plt.show()
"""

> ## Just 'Cause You Can't See It, Doesn't Mean It, Isn't There.
>
> <p style="text-align: right"> <a href="https://www.youtube.com/watch?v=Tm8LGxTLtQk">— One More Light</a> </p>
"""