# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# References:
# https://github.com/bentrevett/pytorch-seq2seq/blob/master/1%20-%20Sequence%20to%20Sequence%20Learning%20with%20Neural%20Networks.ipynb
# -

import torch
from torch import nn, optim
from torchtext.legacy.data import Field, BucketIterator, TabularDataset
from sklearn.model_selection import train_test_split
import numpy as np
import unicodedata
import re
import pandas as pd
import os
import random
import spacy
import shutil

# # !python -m spacy download fr_core_news_sm
# # !python -m spacy download en_core_web_sm
spacy_fr = spacy.load('fr_core_news_sm')
spacy_eng = spacy.load('en_core_web_sm')

path = "D:/Datasets/Eng-French Translation"
os.chdir(path)

df = pd.read_csv('eng_-french.csv')
df.columns = ['english', 'french']


# +
def unicode2Ascii(s):
    return ''.join(c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn')

def normalizeString(s):
    s = unicode2Ascii(s.lower().strip())
    s = re.sub(r"([.!?])", r"\1", s)
    s = re.sub(r"[^a-zA-Z.!?]+", r" ", s)
    return s


# -

df['english'] = df['english'].apply(lambda x: normalizeString(x))
df['french'] = df['french'].apply(lambda x: normalizeString(x))

# +
MAX_LENGTH = 10

def filter_sentence(rows):
    if len(rows['english'].split(' ')) < MAX_LENGTH and len(rows['french'].split(' ')) < MAX_LENGTH:
        return rows
    else:
        return np.nan

df = df.apply(filter_sentence, axis = 'columns')
# -

df_sample = df.dropna().reset_index(drop = True)

np.random.seed(1234)
random.seed(1234)
torch.manual_seed(1234)
n_samples = 1500
df_sample = df_sample.sample(n_samples).reset_index(drop = True)


# +
# tokenizers
def french_tokenizer(text):
    return [tok.text for tok in spacy_fr.tokenizer(text)]

def english_tokenizer(text):
    return [tok.text for tok in spacy_eng.tokenizer(text)]


# +
ENGLISH_TEXT = Field(sequential = True, 
                     tokenize = english_tokenizer, 
                     lower = True,
                     init_token = "<sos>", 
                     eos_token = "<eos>")

FRENCH_TEXT = Field(sequential = True, 
                    tokenize = french_tokenizer, 
                    lower= True,
                    init_token = "<sos>",
                    eos_token = "<eos>")
# -

# train - validation split
train, valid = train_test_split(df_sample, test_size = 0.25, shuffle = True, random_state = 1234)
print("Train : ", train.shape)
print("Valid : ", valid.shape)

# writing train and valid files into folder
if not os.path.exists("inputs"):
    os.mkdir("inputs")
    print("inputs folder created succesfully")
    if not os.path.isfile("/inputs/train.csv"):
        train.to_csv("train.csv", index = False)
        print("train.csv written successfully")
        shutil.move("train.csv", "inputs/train.csv")
        print("train.csv moved successfully")
        
    if not os.path.isfile("/inputs/valid.csv"):
        valid.to_csv("valid.csv", index = False)
        print("valid.csv written successfully")
        shutil.move("valid.csv", "inputs/valid.csv")
        print("valid.csv moved successfully")
else:
    print("Folder already exists")
    train.to_csv("train.csv", index = False)
    print("train.csv written successfully")
    shutil.move("train.csv", "inputs/train.csv")
    print("train.csv moved successfully")
    
    valid.to_csv("valid.csv", index = False)
    print("valid.csv written successfully")
    shutil.move("valid.csv", "inputs/valid.csv")
    print("valid.csv moved successfully")



StarcoderdataJupyterScriptsDedupFiltered