"""
0. ># Preprocessing on Text is important step before any Information Retrieval Application. In this notebook, preprocessing is explained step wise, which are as follows:
> * Combining data from all files into single string using BeautifulSoup
> * Removing Punctuations from text
> * Tokenization using NLTK library
> * Removing Stopwords
> * Stemming shown using both Porter Stemmer & Lemmatization
> * Conclusion/Compare roots formed by Porter Stemmer and those obtained by Lemmatization 
"""
"""
1. > # Importing Libaries
"""
import os,re
from bs4 import BeautifulSoup
import nltk
from nltk.tokenize import word_tokenize
"""
2. >  # **Appending data from all utf files into single string "data"**
"""
files = os.listdir('../input/html-text-files')
data = ''
for i in files:
  obj = open('../input/html-text-files/' + i)
  dat = obj.read()
  soup = BeautifulSoup(dat)
  dat = soup.find('text').text
  data += dat
"""
> # Printing Data
"""
print("Printing only first 2000 words of text for better display:\n\n",data[0:2000])
"""
3. > # Removing Punctuation
"""
data = re.sub('[!\"#$%&\'()*+,-./:;<=>?@[\]^_`{|}~0-9\n]','',data)
print("Printing only first 2000 words of text after removing Puntuation for better display:\n\n",data[0:2000])
"""
4. > # Tokenization
"""
"""
* > Using NLTK library
"""
nltk.download('punkt')
tokens = word_tokenize(data)
print("\n\nTokens are as follows, and I have displayed 100 tokens for better visual display:\n\n",tokens[1:100])
"""
5. > # **Converting all text to Lowercase letters**
"""
tokens = [item.lower() for item in tokens]
print("Resulting tokens are as follows,  i have displayed 100 tokens for better visual display:\n\n",tokens[1:100])
print("\n\nNumber of tokens are : ",len(tokens))
"""
6. > # **Removing stopwords**
"""
from nltk.corpus import stopwords
for i in tokens:
  if i in stopwords.words('english'):
    tokens.remove(i) 
print("Tokens after removing stopwords are, i have displayed 100 tokens for better visual display:\n",tokens[1:100],"\n\nNumber of tokens are :",len(tokens),sep='\n')    
"""
7. > # **Stemming using Porter Stemmer**
"""
stem_porter = []
from nltk.stem import PorterStemmer
porter = PorterStemmer()
for word in tokens:
  stem_porter.append(porter.stem(word))
"""
7. > # **Stemming Using Lemmatization**
"""
stem_lemma = []
nltk.download('wordnet')
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()

for word in tokens:
  stem_lemma.append(lemmatizer.lemmatize(word,pos='v'))
"""
> # **Conclusion what i can make is Lemmatization normalise to roots/stem that do exist in language, while Porter stemmer identifies stem/root which may not exist as a real word in language, language being refered here is English.**
> # Comparsion of roots obtained using Lemmatization and Porter Stemming can be seen below
"""
print("Original_Token      Lemmatization_Token       Porter_Stemmer_Token\n\n")

for word in tokens[1:300]:
  stem_lemma.append(lemmatizer.lemmatize(word,pos='v'))
  print("{0:25}{1:25}{2:25}".format(word,lemmatizer.lemmatize(word,pos='v'),porter.stem(word)))