text: # -*- coding: utf-8 -*-
  """LSTM topic identifier.ipynb
  
  Automatically generated by Colaboratory.
  
  Original file is located at
      https://colab.research.google.com/drive/1922cv48FrmM2zn2PweViYkpShz3moyag
  """
  
  import sys, os, re, csv, codecs, numpy as np, pandas as pd
  import keras
  import tensorflow as tf
  
  from keras.preprocessing.text import Tokenizer
  from keras.preprocessing.sequence import pad_sequences
  from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation,Conv1D,Flatten
  from keras.layers import Bidirectional, GlobalMaxPool1D
  from keras.models import Model
  from keras import initializers, regularizers, constraints, optimizers, layers
  from keras.optimizers import Adam
  from sklearn.model_selection import train_test_split
  
  train = pd.read_csv('/content/drive/My Drive/Module 3/Project/database/topic_identification_data.csv')
  
  train.head()
  
  #preprocessing
  train['comment_text'].fillna('fillna')
  x_train=train['comment_text'].str.lower()
  y_train=train[[ "obscenity", "violence", "verbal_abuse", "identity_hate","hate","offense","neither"]].values
  
  #embedding layer
  embed_size=100
  max_features=20000
  max_len=100
  
  #process the traiing set
  tokenizer= Tokenizer(num_words=max_features,lower= True)
  tokenizer.fit_on_texts(list(x_train))
  tokenized_train=tokenizer.texts_to_sequences(x_train)
  train_x=pad_sequences(tokenized_train,maxlen=max_len)
  
  batch_size=128
  epochs = 2
  
  import matplotlib.pyplot as plt
  plt.style.use('ggplot')
  
  def plot_history(history):
      acc = history.history['accuracy']
      val_acc = history.history['val_accuracy']
      loss = history.history['loss']
      val_loss = history.history['val_loss']
      x = range(1, len(acc) + 1)
  
      plt.figure(figsize=(12, 5))
      plt.subplot(1, 2, 1)
      plt.plot(x, acc, 'b', label='Training acc')
      plt.plot(x, val_acc, 'r', label='Validation acc')
      plt.title('Training and validation accuracy')
      plt.legend()
      plt.subplot(1, 2, 2)
      plt.plot(x, loss, 'b', label='Training loss')
      plt.plot(x, val_loss, 'r', label='Validation loss')
      plt.title('Training and validation loss')
      plt.legend()
  
  # BEST MODEL with 88 % of accuracy
  inp = Input(shape=(max_len,))
  x = Embedding(max_features, embed_size)(inp)
  x = LSTM(50, return_sequences=True)(x)
  x = GlobalMaxPool1D()(x)
  x = Dropout(0.1)(x)
  x = Dense(50, activation="relu")(x)
  x = Dropout(0.1)(x)
  x = Dense(7, activation="sigmoid")(x)
  model = Model(inputs=inp, outputs=x)
  model.compile(loss='categorical_crossentropy',
                    optimizer='adam',
                    metrics=['accuracy'])
  
  history = model.fit(train_x,y_train,epochs=epochs,batch_size=batch_size,validation_split=0.1)
  
  plot_history(history)
  
  # save the model
  model.save('/content/drive/My Drive/Module 3/Project/database/topic_identifier_model.h5')
  
  #test with a sentence
  x_test = "you gay boy"
  
  #preprocess the sentence
  def preprocess_test(x_test):
    """ this function allows to preprocess the test sentence
  
        params :
        x_test (string) : the test message
  
        return :
        test_x (vector) : the processed test message
    """
    x_test = x_test.lower()
    x_test = [x_test]
    tokenized_test=tokenizer.texts_to_sequences(x_test)
    test_x=pad_sequences(tokenized_test,maxlen=max_len)
  
    return test_x
  
  # load the model
  topic_identifier_model =  tf.keras.models.load_model('/content/drive/My Drive/Module 3/Project/database/topic_identifier_model.h5')
  
  #predict the topic
  y_pred=topic_identifier_model.predict(preprocess_test(x_test))
  
  y_pred
  
  label_dict = {0: "obscenity", 1: "violence", 2: "verbal abuse", 3: "identity hate crime", 4: "hate crime", 5: "offense", 6:"neither"}
  
  #get the prediction
  if y_pred.argmax(1)[0] != 6 :
    # if the tweet contains sensitive topics
    print("your tweet may contain sentences that present " + label_dict[y_pred.argmax(1)[0]]+ " with  "+str(y_pred[0][y_pred.argmax(1)[0]]*100) +" of confidence")
  