<filename>Label generation for textual data using un-supervised learning techniques.py
# -*- coding: utf-8 -*-
"""Label generation.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1vWkJR52vOguhorK5RPCbxW6JEeHTz7QZ
"""

#importing libraries
import numpy as np
import pandas as pd
from tensorflow.python.keras.preprocessing.text import Tokenizer
from tensorflow.python.keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense, LSTM, GRU, Embedding, Flatten
from keras.layers.embeddings import Embedding
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.stem.porter import *
import nltk
import re
from bs4 import BeautifulSoup
import gensim
from nltk.tokenize import word_tokenize
import string
from textblob import TextBlob as tb
from collections import Counter
from nltk.stem import WordNetLemmatizer

#importing dataset
from google.colab import files

uploaded = files.upload()

for fn in uploaded.keys():
  print('User uploaded file "{name}" with length {length} bytes'.format(
      name=fn, length=len(uploaded[fn])))

#loading dataset in pandas 
df=pd.DataFrame()
df=pd.read_csv('IMDB Dataset.csv',encoding='utf-8') 
df.head(5)
df['review'][0]

nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')

#some helper function to remove all html stuff from the given string
def strip_html(text):
    soup = BeautifulSoup(text, "html.parser")
    return soup.get_text()

#Removing the square brackets
def remove_between_square_brackets(text):
    return re.sub('\[[^]]*\]', '', text)

#Removing the noisy text
def denoise_text(text):
    text = strip_html(text)
    text = remove_between_square_brackets(text)
    return text

def prepare_data(df):
  #Removing HTML tags
  df['review']=df['review'].apply(denoise_text)
  
  #split dataset
  df['review']=df['review'].apply(lambda x: x.split(" "))
  
  #lower case the text
  df['review']=df['review'].apply(lambda x: [y.lower() for y in x])
  
  #removal of punctuation
  df['review']=df['review'].apply(lambda x: [re.sub(r'[^a-zA-Z#]',' ',y) for y in x])
  
  #removal of stopwords
  stop_words=set(stopwords.words("english"))
  df['review']=df['review'].apply(lambda x: [y for y in x if y not in stop_words])
  df['review']=df['review'].apply(lambda x: " ".join(y for y in x ))
  
  #removing most occurring words
  freq=pd.Series(" ".join(df['review']).split()).value_counts()[:10]
  freq=list(freq.index)
  df['review']=df['review'].apply(lambda x: " ".join([y for y in x.split() if y not in freq]))
  
  #rare words removal
  rare_words=pd.Series(" ".join(df['review']).split()).value_counts()[-10:]
  rare_words=list(rare_words.index)
  df['review']=df['review'].apply(lambda x: " ".join([y for y in x.split() if y not in rare_words]))
  
  #lemmatizing the words to their root words
  lemmatizer=WordNetLemmatizer()
  df['review']=df['review'].apply(lambda x: " ".join([lemmatizer.lemmatize(y,pos="v") for y in x.split() if len(y)>2]))
  df['review']=df['review'].apply(lambda x: " ".join([y for y in x.split() if len(y)>2]))
  return df
  
  
df=prepare_data(df)



#feature selection using TFIDF
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer()
x = vectorizer.fit_transform(df['review'][10000:20000].values.tolist())
x = x.toarray()

#Applying Kmeans clustering for language generation
from sklearn.cluster import KMeans
kmeans1 = KMeans(n_clusters=2, random_state=42).fit(x)

values=np.array(kmeans1.labels_)

#generating texts using cluster centroids
order_centroids = kmeans1.cluster_centers_.argsort()[:, ::-1]
terms = vectorizer.get_feature_names()
for i in range(2):
  print("Cluster %d:" % i, end='')
  for ind in order_centroids[i, :10]:
    print(' %s' % terms[ind], end='')
  print()

def label_changer(x):
  label=-1
  if x=="positive":
    label=1
  elif x=="negative":
    label=0
  return label

#checking the accuracy of clustering
sentiment=df['sentiment'][10000:20000]
sentiment=[label_changer(x) for x in sentiment]
acc=np.sum(sentiment==values)
print ("Accuracy = %d"%((acc/values.shape[0])*100))





