# Initial import statements
import sklearn
import numpy
from __future__ import print_function
from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import Normalizer
from sklearn import metrics
from sklearn.cluster import KMeans, MiniBatchKMeans
import pandas as pd
import warnings
# Suppress warnings from pandas library
warnings.filterwarnings("ignore", category=DeprecationWarning,
module="pandas", lineno=570)

example = ["Machine learning is super fun",
"Python is super, super cool",
"Statistics is cool, too",
"Data science is fun",
"Python is great for machine learning",
"I like football",
"Football is great to watch"]
vectorizer = CountVectorizer(min_df = 1, stop_words = 'english')
dtm = vectorizer.fit_transform(example)
pd.DataFrame(dtm.toarray(),index=example,columns=vectorizer.get_feature_names
()).head(10)

"""
Explanation: ClearAccess IP Proposal
IP Database Semantic Search. We will propose a method to search a database of patent filings and return a listed ranked by semantic relevance. 
Latent Semantic Analysis

Semantic analysis adds structure to unstructured text. It helps uncover hidden relationships and automatically standardises/normalises metadata across large document collections.
Latent Semantic Analysis (LSA) is a framework for analyzing text using matrices
Find relationships between documents and terms within documents
Used for document classification, clustering

Vectorizing text

Most machine-learning and statistical algorithms only work with structured, tabular data
A simple way to add structure to text is to use a document-term matrix
Named Entity Recognition - Extraction of People, Places, Organisations, Times, Email addresses.. from text.

Document-term matrix
We will model each patent document as a row in a matrix:
End of explanation
"""


# Get words that correspond to each column
vectorizer.get_feature_names()

"""
Explanation: Each row represents a document -- this would be patent document. Each column represents a word. So each document is a N-dim (13 in this case) vector.
Each entry equals the number of times the word appears in the document
Note: order and proximity of words in documents is NOT accounted for. Called a "bag of words"
representation.
End of explanation
"""


# Fit LSA. Use algorithm = “randomized” for large datasets
lsa = TruncatedSVD(2, algorithm = 'arpack')
dtm_lsa = lsa.fit_transform(dtm)
dtm_lsa = Normalizer(copy=False).fit_transform(dtm_lsa)

"""
Explanation: Example: "machine" appears once in the first document, "super" appears twice in the second document, and
"statistics" appears zero times in the third document.

Singular value decomposition and LSA
End of explanation
"""


pd.DataFrame(lsa.components_,index = ["component_1","component_2"],columns =
vectorizer.get_feature_names())

"""
Explanation: Each LSA component is a linear combination of words
End of explanation
"""


pd.DataFrame(dtm_lsa, index = example, columns = ["component_1","component_2"])

xs = [w[0] for w in dtm_lsa]
ys = [w[1] for w in dtm_lsa]
xs, ys

#Plot scatter plot of points
%pylab inline
import matplotlib.pyplot as plt
figure()
plt.scatter(xs,ys)
xlabel('First principal component')
ylabel('Second principal component')
title('Plot of points against LSA principal components')
show()

"""
Explanation: Each document is a linear combination of the LSA components
End of explanation
"""


# Plot scatter plot of points with vectors
%pylab inline
import matplotlib.pyplot as plt
plt.figure()
ax = plt.gca()
ax.quiver(0,0,xs,ys,angles='xy',scale_units='xy',scale=1, linewidth = .01)
ax.set_xlim([-1,1])
ax.set_ylim([-1,1])
xlabel('First principal component')
ylabel('Second principal component')
title('Plot of points against LSA principal components')
plt.draw()
plt.show()

"""
Explanation: Geometric picture
End of explanation
"""


#Compute document similarity using LSA components
similarity = np.asarray(numpy.asmatrix(dtm_lsa) * numpy.asmatrix(dtm_lsa).T)
pd.DataFrame(similarity,index=example, columns=example).head(10)

"""
Explanation: We have reduced dimension from 13-dim to 2-dim (and have lost some info)
Similar docs point in similar directions. Dissimilar docs have perpendicular (orthogonal) vectors. "Cosine
similarity"
Can use cosine similarity for search: which doc has the smallest angle with search term?

Document similarity using LSA
End of explanation
"""


