# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# +
from gensim.models import Word2Vec
import numpy as np
import pandas as pd
import os
import sys
import re
sys.path.insert(0,'/workspace/lang-detect/')

from src import utils
# -

data_path = "/workspace/lang-detect/txt/"
dir_list = os.listdir(data_path)
print(dir_list)


def read_data(filepath):
    with open(filepath, "r") as f:
        lines = f.readlines()
        if len(lines) > 1:
            return lines[1].strip("\n")
        return None


# +
# %%time
data, labels = [], []
for dir_name in dir_list:
    files_list = os.listdir(data_path + dir_name)
    for f in files_list:
        sent = read_data(data_path + dir_name + "/" + f)
        if sent:
            data.append(sent)
            labels.append(dir_name)
            
test_data, test_labels = [], []
with open("../../europarl.test", "r") as f:
    for line in f:
        line = line.split()
        test_data.append(" ".join(line[1:]))
        test_labels.append(line[0])

print(test_data[0], test_labels[0])
# -

print("Length of train data", len(data))
print("Length of test data", len(test_data))

# +
# %%time
for i in range(len(data)):
    data[i] = utils.preprocess(data[i])

for i in range(len(test_data)):
    test_data[i] = utils.preprocess(data[i])
# -

print(len(data),len(test_data))

# +
sentences, test_sentences = [], []
for i in range(len(data)):
    x = utils.create_n_gram(data[i], 4)
    sentences.append(x.split())

for i in range(len(test_data)):
    x = utils.create_n_gram(data[i], 4)
    test_sentences.append(x.split())

# +
# %%time
# Create validation set of random 20000 sentences
rand_indices = np.random.choice(len(sentences), 25000)
rand_indices = list(set(rand_indices))[:20000]
print(len(rand_indices))

valid_sentences, valid_labels = [], []

for ind in rand_indices:
    valid_sentences.append(sentences[ind])
    valid_labels.append(labels[ind])

train_sentences = [j for i,j in enumerate(sentences) if i not in rand_indices]    
train_labels = [j for i,j in enumerate(labels) if i not in rand_indices]

print(len(train_sentences), len(train_labels), len(valid_sentences), len(valid_labels))

# +
import pickle

with open("../data/train_sentences.npy","wb") as f:
    pickle.dump(train_sentences, f)
    
with open("../data/train_labels.npy","wb") as f:
    pickle.dump(train_labels, f)

with open("../data/valid_sentences.npy","wb") as f:
    pickle.dump(valid_sentences, f)
    
with open("../data/valid_labels.npy","wb") as f:
    pickle.dump(valid_labels, f)
# -

len(train_sentences)

# ### **Check Point**

# +
from gensim.models import Word2Vec
import numpy as np
import pandas as pd
import os
import sys
import re
sys.path.insert(0,'/workspace/lang-detect/')

from src import utils
import pickle

with open("../data/train_sentences.npy","r") as f:
    train_sentences = pickle.load(f)
    
with open("../data/train_labels.npy","r") as f:
    train_labels = pickle.load(f)

with open("../data/valid_sentences.npy","r") as f:
    valid_sentences = pickle.load(f)
    
with open("../data/valid_labels.npy","r") as f:
    valid_labels = pickle.load(f)
    
print(len(train_sentences), len(train_labels), len(valid_sentences), len(valid_labels))

# +
# Create word_to_index and index_to_word mapping
PAD_ID = 0
UNK_ID = 1

word_to_index, index_to_word, word_freq = {}, {}, {}

index = 2
for sent in train_sentences:
    for token in sent:
        if token not in word_to_index:
            word_freq[token] = 1
            word_to_index[token] = index
            index_to_word[index] = token
            index += 1
        else:
            word_freq[token] += 1

print("Vocabulary size", index-1)
# -

classes = {}
for ind, c in enumerate(list(set(train_labels))):
    classes[c] = ind
print(classes)

less_freq = 0
for k,v in word_freq.items():
    if v <= 1:
        less_freq += 1
print(less_freq)


# +
# Change words to their index
def get_indexed_data(sentences, data_type=None):
    data_x = []
    for sent in sentences:
        x = []
        for token in sent:
            if token in word_to_index:
                if (data_type == "train") and (word_freq[token] == 1):
                    x.append(UNK_ID)
                else:
                    x.append(word_to_index[token])
            else:
                x.append(UNK_ID)
        data_x.append(x)
    return data_x

def get_indexed_label(labels):
    data_y = [classes[x] for x in labels]
    return data_y

train_x = get_indexed_data(train_sentences, "train")
valid_x = get_indexed_data(valid_sentences)
test_x = get_indexed_data(test_sentences)

train_y = get_indexed_label(train_labels)
valid_y = get_indexed_label(valid_labels)
test_y = get_indexed_label(test_labels)

print(len(train_x),len(valid_x),len(test_x))
print(len(train_y),len(valid_y),len(test_y))


# +
# Check distribution of length of sentences
def get_dist(x):
    lens = {}
    for sent in x:
        if len(sent) in lens:
            lens[len(sent)] += 1
        else:
            lens[len(sent)] = 1
    return lens
        
train_lens = get_dist(train_x)
valid_lens = get_dist(valid_x)
test_lens = get_dist(test_x)
# -

import matplotlib.pyplot as plt
# %matplotlib inline  

# +
fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(15,8))

plt.subplot(221)
plt.bar(train_lens.keys(), train_lens.values())
plt.title("Training Set Distribution")
plt.xlabel('Length of Sentences')
plt.ylabel('No. of Sentences')

plt.subplot(222)
plt.bar(valid_lens.keys(), valid_lens.values())
plt.title("Validation Set Distribution")
plt.xlabel('Length of Sentences')
plt.ylabel('No. of Sentences')

plt.subplot(223)
plt.bar(test_lens.keys(), test_lens.values())
plt.title("Testing Set Distribution")
plt.xlabel('Length of Sentences')
plt.ylabel('No. of Sentences')
plt.figure(figsize=(10,5))

fig.tight_layout()
plt.show()


# -

# Training, Validation and Testing set have similar distribution. Also most of the sentences have length less than 250.

# +
# in train check number of sentences with length more than 100, 200, 300
def check_sent_length_buckets(len_dists):
    n_100, n_200, n_250, n_300, n_500 = 0, 0, 0, 0, 0
    for k,v in len_dists.items():
        if k >= 500:
            n_500 += v
        if k >= 300:
            n_300 += v
        if k >= 200:
            n_200 += v
        if k >= 250:
            n_250 += v
        if k >= 100:
            n_100 += v
    print(n_100, n_200, n_250, n_300, n_500)

check_sent_length_buckets(train_lens)
check_sent_length_buckets(valid_lens)
check_sent_length_buckets(test_lens)
# -

# **There are some very long sentences but I don't want to loose information. What I am going to do is, truncate the sentences with length more than 250 and create a new sentence from index 251 to 500 and so on. This will be done only for train. Since I have included 4-gram tokens only in this data, some noise might get added at the start and end of new sentences, but the model should not get affected by that.**

# +
train_x_trunc, train_y_trunc = [], []

max_len = 250

for ind, sent in enumerate(train_x):
    while(len(sent) > max_len):
        train_x_trunc.append(sent[:max_len])
        train_y_trunc.append(train_y[ind])
        sent = sent[max_len:]
    train_x_trunc.append(sent)
    train_y_trunc.append(train_y[ind])

print(len(train_x_trunc), len(train_y_trunc))
# -

# ##### **Create batches**

train_lens = get_dist(train_x)

buckets = [10*x for x in range(1,26)]
buckets_data_sum = {}
for k, v in train_lens.items():
    for x in buckets:
        if k <= x:
            if x in buckets_data_sum:
                buckets_data_sum[x] += v
            else:
                buckets_data_sum[x] = v
            break
buckets_data_sum

sum(buckets_data_sum.values())

# +
from collections import defaultdict

# Create batch data
def create_batches(data_x, data_y):
    batch_data = defaultdict(list)
    batch_label = defaultdict(list)

    for ind, sent in enumerate(data_x):
        for x in buckets:
            if len(sent) <= x:
                sent += [PAD_ID]*(x - len(sent))
                batch_data[x].append(sent)
                batch_label[x].append(data_y[ind])
                break
    
    return batch_data, batch_label

train_batch_data, train_batch_labels = create_batches(train_x_trunc, train_y_trunc)
valid_batch_data, valid_batch_labels = create_batches(valid_x, valid_y)
# -

train_batch_data[10][:10]


# +
# Convert to numpy array each key
def to_numpy_array(batch_data, batch_label):
    for key in batch_data:
        batch_data[key] = np.array(batch_data[key])
        batch_label[key] = np.array(batch_label[key])
    return batch_data, batch_label

train_batch_data, train_batch_labels = to_numpy_array(train_batch_data, train_batch_labels)
valid_batch_data, valid_batch_labels = to_numpy_array(valid_batch_data, valid_batch_labels)
test_data, test_labels = np.array(test_x), np.array(test_y)

# +
import pickle

train = {'data':train_batch_data, 'labels':train_batch_labels}
valid = {'data':valid_batch_data, 'labels':valid_batch_labels}
test = {'data':test_data, 'labels':test_labels}

with open('../data/train_batch.npy', 'w') as f:
    pickle.dump(train, f)

with open('../data/valid_batch.npy', 'w') as f:
    pickle.dump(valid, f)
    
with open('../data/test_data.npy', 'w') as f:
    pickle.dump(test, f)
# -

# Create embedding matrix
wordvec = Word2Vec.load("../word-vectors/word2vec")
wordvec.wv.most_similar("the")

wordvec.wv.get_vector("the")

# Tokens are indexed from 2. '0' is for the padding and '1' for unknown token. Vector for 0 will be zero-vector and for 1, a random vector.

vocab_size, embed_size = wordvec.wv.vectors.shape

sorted_word_to_index = sorted(zip(word_to_index.keys(), word_to_index.values()), key=lambda x: x[1])
sorted_word_to_index[0]

# +
embedding_matrix = []
embedding_matrix.append([0.]*embed_size)
embedding_matrix.append(np.random.uniform(-3, 3, embed_size))

for key, index in sorted_word_to_index:
    embedding_matrix.append(wordvec.wv.get_vector(key))
# -

print(len(embedding_matrix), len(embedding_matrix[0]))

# +
embedding_matrix = np.array(embedding_matrix)

import pickle
with open('../data/embedding_matrix.npy', 'w') as f:
    pickle.dump(embedding_matrix, f)
# -

with open('../data/embedding_matrix.npy', 'r') as f:
    ld = pickle.load(f)
ld.shape



StarcoderdataJupyterScriptsDedupFiltered