# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="tyn3eevxW6Na"
# # Text classification: We are going to developp a model able to classify text in 6 categories.

# + [markdown] id="hekaCsWFQMdL"
# # Choose directories

# + id="GPGRNxmxPi33"
"""

data_path : concerns the path of the .csv file used as training set
models_folder : referes to the folder where the models obtain after each epochs will store.
main_folder : refers to folder containing your Train.csv file, the of Dataset folder that 
N_EPOCHS : is the number of iterations

"""

data_path = "drive/MyDrive/Dataset/text classification/Train.csv"
models_folder = "drive/MyDrive/Models/text classification/"
main_folder = "drive/MyDrive/Dataset/text classification/"
N_EPOCHS = 12

# + [markdown] id="yKoL2hXIhqtg"
# ## Installation spacy==3.1.0 et fr_code_news_sm pour spacy

# + id="D5lxw_dihjZc"
# %pip install bitarray fastBPE hydra-core omegaconf regex requests sacremoses subword_nmt gradio spacy==3.1.0
# !python -m spacy download fr_core_news_sm

# + [markdown] id="BRVLDCZdW6Nj"
# ## Import

# + id="KbVEnLjLW6Nk"
import os
import torch
from torchtext.legacy import data
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import random


# + colab={"base_uri": "https://localhost:8080/"} id="av5SprrHt1MN" outputId="9260f9d5-d977-42a3-debf-9e67daf05a0b"
# If you are usingcollab

from google.colab import drive
drive.mount('/content/drive')

# + [markdown] id="7dzZUWe8W6No"
# ## Reproducibility

# + id="7XB-8h_hW6Nq"
SEED = 2021

def reproducibility(seed=2021):
    
    # seed for random, os, numpy and torch librayr
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    
    # seed fo cuda
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

reproducibility(SEED)

# + [markdown] id="m3cPhuljW6Nr"
# ## EDA

# + id="Mqzo-wbOW6Nt"
donnees = pd.read_csv(data_path, index_col="Tweet_ID")

sns.set_style("dark")
sns.set_context("notebook")
plt.figure(figsize=(8, 5))
plt.subplot(111)
sns.countplot(data=donnees, x="type")
plt.xticks(rotation=90)
plt.legend(loc="upper right")
plt.show()

# + id="JBS_O51jW6Nu"
num_data_per_class = donnees["type"].value_counts()
num_data_per_class

# + id="rk89AiZ2W6Nw"
percentage_data_per_class = 100*donnees["type"].value_counts()/donnees.shape[0]
percentage_data_per_class

# + id="VZTU8IAFW6Ny"
1 / (100*donnees["type"].value_counts()/donnees.shape[0])

# + [markdown] id="vj2hhLkhW6Nz"
# ## Resampling

# + id="1Daq78J_W6N0"
labels = ["sexual_violence", "Physical_violence", "emotional_violence",
         "economic_violence", "Harmful_Traditional_practice"]

# + [markdown] id="_vj0r1PtW6N1"
# ## For generate french dataset

# + id="Ftn_a7iELKJQ"
# Load an En-Fr Transformer model trained on WMT'14 data :
en2fr = torch.hub.load('pytorch/fairseq:v0.10.2', 'transformer.wmt14.en-fr', tokenizer='moses', bpe='subword_nmt')

# Use the GPU (optional):
en2fr.cuda()

# + colab={"background_save": true} id="EH0LxUw-W6N4"
import math
percent = 0.7

new_train_data = pd.concat([donnees.loc[donnees["type"]==t, :].sample(130) for t in labels]
                , axis=0)

new_valid_data = pd.concat([donnees.loc[donnees["type"]==t, :].sample(58) for t in labels]
                , axis=0)

new_train_data["tweet"] = pd.DataFrame(new_train_data["tweet"].apply(lambda x:  en2fr.translate(x)))
new_valid_data["tweet"] = pd.DataFrame(new_valid_data["tweet"].apply(lambda x:  en2fr.translate(x)))

new_train_data.to_csv(main_folder+"new_train.csv")
new_valid_data.to_csv(main_folder+"new_valid.csv")

# + [markdown] id="J8r72yIIW6N5"
# ## Preprocessing

# + id="Fh84IT4OW6N5"
# Now, let us see how to preprocess the text using field objects.
# There are 2 different types of field objects – Field and LabelField.
# Field concern the preprocessing tehcnics to apply on text ans LabelField concern ones toapply for Label.

TEXT = data.Field(tokenize='spacy', tokenizer_language='fr', batch_first=True, include_lengths=True)
LABEL = data.LabelField()

# This variable is used to read file data.
fields = [('Tweet_ID',None), ('tweet',TEXT), ('type', LABEL)]

# Loading custom dataset    
train_data=data.TabularDataset(path = 'drive/MyDrive/Dataset/text classification/new_train.csv'
                                  ,format = 'csv', fields = fields, skip_header = True)
valid_data=data.TabularDataset(path = 'drive/MyDrive/Dataset/text classification/new_valid.csv'
                                  ,format = 'csv', fields = fields, skip_header = True)

# Print preprocessed text
print(vars(train_data.examples[0]))

# + [markdown] id="MAuZO8iFW6N6"
# ## Split data to train and validation set 

# + id="gZnpuDhFW6N7"
# Check whether cuda is available
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')  

# Set batch size
BATCH_SIZE = 64

# Load iterators
train_iterator, valid_iterator = data.BucketIterator.splits(
    (train_data, valid_data),
    batch_size = BATCH_SIZE,
    sort_key = lambda x: len(x.tweet),
    sort_within_batch=True,
    device = device)

# + [markdown] id="0WbrDG-XW6N7"
# ## Preparing input and output sequences

# + colab={"base_uri": "https://localhost:8080/"} id="7agRN0dCW6N9" outputId="546d82cd-326b-4988-a989-24132b5873ad"
# Initialize glove embeddings
TEXT.build_vocab(train_data, min_freq=3, vectors = "glove.6B.100d")  
LABEL.build_vocab(train_data)

# No. of unique tokens in text
print("Size of TEXT vocabulary:",len(TEXT.vocab))

# No. of unique tokens in label
print("Size of LABEL vocabulary:",len(LABEL.vocab))

# Commonly used words
print("Commonly used words", TEXT.vocab.freqs.most_common(10))  

# Word dictionary
print("Word dictionary", TEXT.vocab.stoi.items())   

# + colab={"base_uri": "https://localhost:8080/"} id="UtFZJAVIKGJ7" outputId="d3c7eab2-3f51-4734-f5a1-0d588e5f5453"
help(TEXT.build_vocab)

# + [markdown] id="X3GK-42KW6N-"
# ## Build the model

# + id="YjhJaGDoW6N-"
import torch.nn as nn

class classifier(nn.Module):
    
    #define all the layers used in model
    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers, 
                 bidirectional, dropout):
        
        #Constructor
        super().__init__()          
        
        #embedding layer
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        
        #lstm layer
        self.lstm = nn.LSTM(embedding_dim, 
                           hidden_dim, 
                           num_layers=n_layers, 
                           bidirectional=bidirectional, 
                           dropout=dropout,
                           batch_first=True)
        
        #dense layer
        self.fc = nn.Linear(hidden_dim * 2, output_dim)
        
        #activation function
        self.act = nn.Sigmoid()
        
    def forward(self, text, text_lengths):
        
        #text = [batch size,sent_length]
        embedded = self.embedding(text)
        #embedded = [batch size, sent_len, emb dim]
      
        #packed sequence
        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths.to("cpu"), batch_first=True)
        
        packed_output, (hidden, cell) = self.lstm(packed_embedded)
        #hidden = [batch size, num layers * num directions,hid dim]
        #cell = [batch size, num layers * num directions,hid dim]
        
        #concat the final forward and backward hidden state
        hidden = torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1)
                
        #hidden = [batch size, hid dim * num directions]
        dense_outputs=self.fc(hidden)

        #Final activation function
        outputs=self.act(dense_outputs)
        
        return outputs


# + id="n2b1jTd6W6N_"
#define hyperparameters
size_of_vocab = len(TEXT.vocab)
embedding_dim = 100
num_hidden_nodes = 32
num_output_nodes = 5
num_layers = 2
bidirection = True
dropout = 0.2

#instantiate the model
model = classifier(size_of_vocab, embedding_dim, num_hidden_nodes,num_output_nodes, num_layers, 
                   bidirectional = True, dropout = dropout)

# + colab={"base_uri": "https://localhost:8080/"} id="yDbyDRL5W6N_" outputId="2c8064ee-59ea-42f1-dc3e-5a9413d17d0a"
#architecture
print(model)

#No. of trianable parameters
def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)
    
print(f'The model has {count_parameters(model):,} trainable parameters')

#Initialize the pretrained embedding
pretrained_embeddings = TEXT.vocab.vectors
model.embedding.weight.data.copy_(pretrained_embeddings)

print(pretrained_embeddings.shape)

# + id="OWYVyttfW6OA"
import torch.optim as optim

#define optimizer and loss
optimizer = optim.Adam(model.parameters())
criterion = nn.BCELoss()#weight=torch.tensor([0.012145, 0.066683, 0.609063, 1.827189, 2.109043])) 

#define metric
def binary_accuracy(preds, y):
    #round predictions to the closest integer
    rounded_preds = torch.round(preds)
    
    correct = (rounded_preds == y).float() 
    acc = correct.sum() / len(correct)
    return acc
    
#push to cuda if available
model = model.to(device)
criterion = criterion.to(device)


# + id="3XJt8N8FW6OB"
def train(model, iterator, optimizer, criterion):
    
    #initialize every epoch 
    epoch_loss = 0
    epoch_acc = 0
    
    #set the model in training phase
    model.train()  
    
    for batch in iterator:
        
        #resets the gradients after every batch
        optimizer.zero_grad()   
        
        #retrieve text and no. of words
        text, text_lengths = batch.tweet   
        #convert to 1D tensor
        predictions = model(text, text_lengths)
        
        #compute the loss
        batch_size = batch.type.shape[0]
        target = torch.zeros((batch_size, num_output_nodes))
        target[[i for i in range(batch_size)], batch.type.to("cpu").numpy()] = 1.0
        target = target.to(device)
        
        loss = criterion(predictions, target)        
        
        #compute the binary accuracy
        #acc = binary_accuracy(predictions, batch.label)   
        
        #backpropage the loss and compute the gradients
        loss.backward()       
        
        #update the weights
        optimizer.step()      
        
        #loss and accuracy
        epoch_loss += loss.item()  
        #epoch_acc += acc.item()    
        
    return epoch_loss / len(iterator)#, epoch_acc / len(iterator)


# + id="T6twCcmnW6OC"
def evaluate(model, iterator, criterion):
    
    #initialize every epoch
    epoch_loss = 0
    epoch_acc = 0

    #deactivating dropout layers
    model.eval()
    
    #deactivates autograd
    with torch.no_grad():
    
        for batch in iterator:
        
            #retrieve text and no. of words
            text, text_lengths = batch.tweet
            
            #convert to 1d tensor
            predictions = model(text, text_lengths)
            
            #compute the loss
            batch_size = batch.type.shape[0]
            target = torch.zeros((batch_size, num_output_nodes))
            target[[i for i in range(batch_size)], batch.type.to("cpu").numpy()] = 1.0
            target = target.to(device)
            
            #compute loss and accuracy
            loss = criterion(predictions, target)        
            #acc = binary_accuracy(predictions, batch.label)
            
            #keep track of loss and accuracy
            epoch_loss += loss.item()
            #epoch_acc += acc.item()
        
    return epoch_loss / len(iterator)#, epoch_acc / len(iterator)


# + colab={"base_uri": "https://localhost:8080/"} id="S1Wgv9LdW6OD" outputId="41042b1c-c357-4f9e-8b7a-2bb09a06921b"
best_valid_loss = float('inf')

train_loss = np.zeros((1, N_EPOCHS))
valid_loss = np.zeros((1, N_EPOCHS))

for epoch in range(N_EPOCHS):
     
    #train the model
    train_loss[0, epoch] = train(model, train_iterator, optimizer, criterion)
    
    #evaluate the model
    valid_loss[0, epoch] = evaluate(model, valid_iterator, criterion)
    
    #save the best model
    if valid_loss[0, epoch] < best_valid_loss:
        best_valid_loss = valid_loss[0, epoch]
        torch.save(model.state_dict(), models_folder+'saved_weights('+str(epoch)+').pt')
    
    print(f"EPOCH {epoch}%")
    print(f'\tTrain Loss: {train_loss[0, epoch]:.3f}%')# | Train Acc: {train_acc*100:.2f}%')
    print(f'\t Val. Loss: {valid_loss[0, epoch]:.3f}%')# |  Val. Acc: {valid_acc*100:.2f}%')


# + colab={"base_uri": "https://localhost:8080/", "height": 301} id="1Lx40YzdW6OD" outputId="1a7ddf7d-74c2-4a73-9784-00ff661c3738"
# Plot the complexity graph
plt.plot(train_loss[0,:], label = "train")
plt.plot(valid_loss[0,:], label = "validation")
plt.xlabel('epoch')
# Set the y axis label of the current axis.
plt.ylabel('error')
# Set a title of the current axes.
plt.title('Complexity graph')
# show a legend on the plot
plt.legend()
# Display a figure.
plt.show()

# + id="9mztMo3YW6OE"
#load weights
path=models_folder+'saved_weights('+str(N_EPOCH-1)+').pt'
model.load_state_dict(torch.load(path));
model.eval();

#inference 
import spacy
nlp = spacy.load("fr_core_news_sm")

CLASS = ["Sexual violence", "Physical violence", "Emotional Violence",
         "Economic Violence", "Harmful traditional practice"]

def predict(sentence):
    tokenized = [tok.text for tok in nlp.tokenizer(sentence)]  #tokenize the sentence 
    indexed = [TEXT.vocab.stoi[t] for t in tokenized]          #convert to integer sequence
    
    length = len(indexed)
    length_tensor = torch.LongTensor([length]).to('cpu')
    
    my_tensor = torch.LongTensor(indexed).to(device)           #convert to tensor
    my_tensor = torch.reshape(my_tensor, (1, length))          #reshape in form of batch,no. of words
    
    prediction = model(my_tensor, length_tensor)               #prediction 
    i = prediction.argmax().to('cpu').numpy()
    
    return CLASS[i]


# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="LGrzvVHvW6OE" outputId="359809e2-71b3-42ce-8fd0-126b111355f7"
# make predictions
predict("IL m'a oubligé à coucher avec lui")

StarcoderdataJupyterScriptsDedupFiltered