text: #------------------------------------------------------------------------------------------------------------#
  #Chapter 3 - Decorrelating your data and dimension reduction
  
  
  #------------------------------------------------------------------------------------------------------------#
  
  #Correlated data in nature
  # Perform the necessary imports
  import matplotlib.pyplot as plt
  from scipy.stats import pearsonr
  
  # Assign the 0th column of grains: width
  width = grains[:,0]
  
  # Assign the 1st column of grains: length
  length = grains[:,1]
  
  # Scatter plot width vs length
  plt.scatter(width, length)
  plt.axis('equal')
  plt.xlabel('width')
  plt.ylabel('length')
  plt.title('I suspect that width and length will be correlated.')
  plt.show()
  
  # Calculate the Pearson correlation
  correlation, pvalue = pearsonr(width,length)
  
  # Display the correlation
  print(correlation)
  
  
  #------------------------------------------------------------------------------------------------------------#
  #Decorrelating the grain measurements with PCA
  # Import PCA
  from sklearn.decomposition import PCA
  
  # Create PCA instance: model
  model = PCA()
  
  # Apply the fit_transform method of model to grains: pca_features
  pca_features = model.fit_transform(grains)
  
  # Assign 0th column of pca_features: xs
  xs = pca_features[:,0]
  
  # Assign 1st column of pca_features: ys
  ys = pca_features[:,1]
  
  # Scatter plot xs vs ys
  plt.scatter(xs, ys)
  plt.axis('equal')
  plt.show()
  
  # Calculate the Pearson correlation of xs and ys
  correlation, pvalue = pearsonr(xs, ys)
  
  # Display the correlation
  print(correlation)
  
  
  
  #------------------------------------------------------------------------------------------------------------#
  
  #The first principal component
  # Make a scatter plot of the untransformed points
  plt.scatter(grains[:,0], grains[:,1])
  
  # Create a PCA instance: model
  model = PCA()
  
  # Fit model to points=points
  points=model.fit(grains)
  
  # Get the mean of the grain samples: mean
  mean = model.mean_
  
  # Get the first principal component: first_pc
  first_pc = model.components_[0,:]
  
  # Plot first_pc as an arrow, starting at mean
  plt.arrow(mean[0],mean[1], first_pc[0], first_pc[1], color='red', width=0.01)
  
  # Keep axes on same scale
  plt.axis('equal')
  plt.show()
  
  #------------------------------------------------------------------------------------------------------------#
  #Variance of the PCA features
  # Perform the necessary imports
  from sklearn.decomposition import PCA
  from sklearn.preprocessing import StandardScaler
  from sklearn.pipeline import make_pipeline
  import matplotlib.pyplot as plt
  
  # Create scaler: scaler
  scaler = StandardScaler()
  
  # Create a PCA instance: pca
  pca = PCA()
  
  # Create pipeline: pipeline
  pipeline = make_pipeline(scaler,pca)
  
  # Fit the pipeline to 'samples'
  pipeline.fit(samples)
  
  # Plot the explained variances
  features = range(pca.n_components_)
  plt.bar(features, pca.explained_variance_)
  plt.xlabel('PCA feature')
  plt.ylabel('variance')
  plt.xticks(features)
  plt.show()
  
  
  #------------------------------------------------------------------------------------------------------------#
  
  #Dimension reduction of the fish measurements
  # Import PCA
  from sklearn.decomposition import PCA
  
  # Create a PCA instance with 2 components: pca
  pca = PCA(n_components=2)
  
  # Fit the PCA instance to the scaled samples
  pca.fit(scaled_samples)
  
  # Transform the scaled samples: pca_features
  pca_features = pca.transform(scaled_samples)
  
  # Print the shape of pca_features
  print(pca_features.shape)
  
  
  #------------------------------------------------------------------------------------------------------------#
  #term frequency (tf) and inverse document frequency (idf)
  "For this, use the TfidfVectorizer from sklearn. It transforms a list of documents into a word frequency array, which it outputs as a csr_matrix. It has fit() and transform() methods like other sklearn objects."
  
  # Import TfidfVectorizer
  from sklearn.feature_extraction.text import TfidfVectorizer
  
  # Create a TfidfVectorizer: tfidf
  tfidf = TfidfVectorizer() 
  
  # Apply fit_transform to document: csr_mat
  csr_mat = tfidf.fit_transform(documents)
  
  # Print result of toarray() method
  print(csr_mat.toarray())
  
  # Get the words: words
  words = tfidf.get_feature_names()
  
  # Print words
  print(words)
  
  
  
  #------------------------------------------------------------------------------------------------------------#
  #Clustering Wikipedia part I
  # Perform the necessary imports
  from sklearn.decomposition import TruncatedSVD
  from sklearn.cluster import KMeans
  from sklearn.pipeline import make_pipeline
  
  # Create a TruncatedSVD instance: svd
  svd = TruncatedSVD(n_components=50)
  
  # Create a KMeans instance: kmeans
  kmeans = KMeans(n_clusters=6)
  
  # Create a pipeline: pipeline
  pipeline = make_pipeline(svd,kmeans)
  
  
  #------------------------------------------------------------------------------------------------------------#
  
  #Clustering Wikipedia part II
  # Import pandas
  import pandas as pd
  
  # Fit the pipeline to articles
  pipeline.fit(articles)
  
  # Calculate the cluster labels: labels
  labels = pipeline.predict(articles)
  
  # Create a DataFrame aligning labels and titles: df
  df = pd.DataFrame({'label': labels, 'article': titles})
  
  # Display df sorted by cluster label
  print(df.sort_values('label'))
  
  
  #------------------------------------------------------------------------------------------------------------#
  
  
  #------------------------------------------------------------------------------------------------------------#
  
  
  
  #------------------------------------------------------------------------------------------------------------#
  
  
  
  
  #------------------------------------------------------------------------------------------------------------#
  
  
  
  
  
  #------------------------------------------------------------------------------------------------------------#
  
  
  
  
  
  #------------------------------------------------------------------------------------------------------------#
  
  
  
  
  
  #------------------------------------------------------------------------------------------------------------#
  
  
  
  
  
  #------------------------------------------------------------------------------------------------------------#
  