"""
<h1><center>EDA + Submission Code</center></h1>
                                                      
<center><img src = "https://miro.medium.com/max/576/1*NzmIA9eUULy-hNyVOF9g5A.png" width = "750" height = "500"/></center>
"""
"""
### <center>If you find this notebook useful and resourceful, do leave behind a upvote. I will be updating this notebook on a regular basis, so please check back once new version comes up.☠️☠️</center>
"""
"""
## Evaluation Criteria
We will be using mean F score on the data set. F1-Score :-
> In statistical analysis of binary classification, the F-score or F-measure is a measure of a test's accuracy. It is calculated from the precision and recall of the test, where the precision is the number of true positive results divided by the number of all positive results, including those not identified correctly, and the recall is the number of true positive results divided by the number of all samples that should have been identified as positive. Precision is also known as positive predictive value, and recall is also known as sensitivity in diagnostic binary classification.

<center><img src = "https://miro.medium.com/max/1530/1*wUdjcIb9J9Bq6f2GvX1jSA.png" width = "550" height = "150"/></center>

<center><img src = "https://miro.medium.com/max/1872/1*pOtBHai4jFd-ujaNXPilRg.png" width = "750" height = "250"/></center>

> In pattern recognition, information retrieval and classification (machine learning), precision (also called positive predictive value) is the fraction of relevant instances among the retrieved instances, while recall (also known as sensitivity) is the fraction of relevant instances that were retrieved. Both precision and recall are therefore based on relevance.

> Suppose a computer program for recognizing dogs (the relevant element) in photographs identifies eight dogs in a picture containing ten cats and twelve dogs, and of the eight it identifies as dogs, five actually are dogs (true positives), while the other three are cats (false positives). Seven dogs were missed (false negatives), and seven cats were correctly excluded (true negatives). The program's precision is then 5/8 (true positives / all positives) while its recall is 5/12 (true positives / relevant elements).

where :-
* TP = True Positive
* FP = False Positive
* TN = True Negative
* FN = False Negative
"""
from IPython.core.display import display, HTML, Javascript

def ApplyCustomCSS():
    return HTML("<style>"+open("../input/customcss/custom_kaggle_forCommit.css", "r").read()+"</style>")

ApplyCustomCSS()
"""
If you wanna know more about modifying how a notebook look then please visit the Kaggle topic here [Click Here](https://www.kaggle.com/discussion/230082)
"""
"""
## Data Set Information
[train/test].csv - the training set metadata. Each row contains the data for a single posting. Multiple postings might have the exact same image ID, but with different titles or vice versa.

posting_id - the ID code for the posting.
image - the image id/md5sum.
image_phash - a perceptual hash of the image.
title - the product description for the posting.
label_group - ID code for all postings that map to the same product. Not provided for the test set.

train/test images - the images associated with the postings.

### Importing Dependencies
"""
import numpy as np
import pandas as pd
import warnings as w
from scipy import spatial
from tqdm.notebook import tqdm
import random, math, cv2, os, string, re, gc

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
import tensorflow as tf
import tensorflow_hub as hub
from tensorflow.keras import layers as L
import tensorflow.keras as K
from sklearn.model_selection import train_test_split
import cudf, cuml, cupy
from cuml.neighbors import NearestNeighbors
from cuml.feature_extraction.text import TfidfVectorizer

import seaborn as sns
import matplotlib.pyplot as plt
from wordcloud import WordCloud, STOPWORDS
%matplotlib inline
sns.set(style="whitegrid")


w.filterwarnings('ignore')

TRAIN_BASE = '../input/shopee-product-matching/train_images/'
TEST_BASE = '../input/shopee-product-matching/test_images/'
SEED = 100
IMG_SIZE = 444
EPOCHS = 2

sample_sub = pd.read_csv("../input/shopee-product-matching/sample_submission.csv")
test = pd.read_csv("../input/shopee-product-matching/test.csv")
train = pd.read_csv("../input/shopee-product-matching/train.csv")

# train = pd.concat([train, train, train[:2000]], axis = 0)
# train.reset_index(drop = True, inplace = True)
# print("shape: ", train.shape)

train.head()
IS_GPU_AVAIL = tf.config.experimental.list_physical_devices('GPU')
if IS_GPU_AVAIL:
    try:
        tf.config.experimental.set_virtual_device_configuration(IS_GPU_AVAIL[0],
            [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024*6.0)])
        lgpu = tf.config.experimental.list_logical_devices('GPU')
        
    except RuntimeError as ex:
        print(ex)
        
print(f'Tensorflow GPU space 6GB GPU RAM')
print('RAPIDS GPU space 10GB GPU RAM')
def seed_everything(seed):
    random.seed(seed)
    np.random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    tf.random.set_seed(seed)
    
def process_data(df):
    label_to_encoded = {idx:item for idx,item in enumerate(df.label_group.unique())}
    encoded_to_label = {item:idx for idx,item in enumerate(df.label_group.unique())}
    classes = df.label_group.nunique()
    return label_to_encoded, encoded_to_label, classes

seed_everything(SEED)
label_to_encoded, encoded_to_label, NUM_CLASSES = process_data(train)

for col in train.columns:
    print(f"Number of unique {col} entries : ", train[col].nunique(), " against the dataset of size ", train.shape)
"""
## Optimum Image Size

Finding the average of height and width of all the images. As in the dataset the images size vary from 100x100 to something like 5000x5000, which is a vast range so taking the average should suffice because if a particular image is trimmed down too much the pixels start overlapping and is a image expanded to much then one pixel covers a very large portion 
"""
# size = set()
# for i in train['image'].values:
#     img = cv2.imread(TRAIN_BASE+i)
#     sh = tuple(img.shape)
#     size.add(sh)

# sizel = list(size)

# ## sorting the size array based on width
# sizel = sorted(sizel, key = lambda x: x[1])
# IMG_SIZEavg = int(np.average([i for i in sizel]))

# print("Taking the average of all the images sizes ", IMG_SIZEavg)
# sizel[:5], sizel[-5:]
# gc.collect()
for IMG_IDX in [1,2]: 
    temp_img = cv2.imread(TRAIN_BASE+train['image'].values[IMG_IDX])
    ct = cv2.resize(temp_img, (IMG_SIZE, IMG_SIZE))

    print(temp_img.shape, " => ", ct.shape)

    plt.subplot(1,2,1)
    plt.imshow(temp_img)
    plt.subplot(1,2,2)
    plt.imshow(ct)
    plt.show()
if test.shape[0] == 3:
    for IMG_IDX in range(3): 
        temp_img = cv2.imread(TEST_BASE+test['image'].values[IMG_IDX])
        ct = cv2.resize(temp_img, (IMG_SIZE, IMG_SIZE))

        print(temp_img.shape, " => ", ct.shape)
        print("Title : ", test['title'].values[IMG_IDX])
        plt.imshow(temp_img)
        plt.imshow(ct)
        plt.show()
"""
## Now Lets do some digging

Getting to know the data from the features prespective.

### Label Sorting and Discovery
"""
best10 = train['label_group'].value_counts().index.tolist()[:10]
best10_vals = train['label_group'].value_counts().tolist()[:10]

plt.figure(figsize=(10, 6))
sns.barplot(x=best10, y=best10_vals, palette="vlag")
plt.xticks(rotation=45)
plt.xlabel("Label_Group")
plt.ylabel("Num of Images")
plt.title("Top 10 Labels by Images Count")
plt.show()
worst10 = train['label_group'].value_counts().index.tolist()[-10:]
worst10_vals = train['label_group'].value_counts().tolist()[-10:]

plt.figure(figsize=(10, 6))
sns.barplot(x=worst10, y=worst10_vals, palette="rocket")
plt.xticks(rotation=45)
plt.xlabel("Label_Group")
plt.ylabel("Num of Images")
plt.title("10 Worst Labels by Images Count")
plt.show()
for item in train.label_group.unique()[3001:3004]: 
    df_temp = train[train.label_group == item]
    
    cols = df_temp.shape[0]
    fig = plt.figure(figsize=(10,20))
    for idx,(index,row) in enumerate(df_temp.iterrows()):
        temp_img = cv2.imread(TRAIN_BASE+row.image)
        ct = cv2.resize(temp_img, (IMG_SIZE, IMG_SIZE))
        plt.subplot(1,cols,idx+1)
        plt.imshow(temp_img)
        plt.axis('off')
    
    print('\033[1m' + '\033[36m'+"Group Label"+'\033[0m', item)
    print('\033[1m' + '\033[32m'+"Number of items in Group "+'\033[0m', cols)
    print('\033[1m' + '\033[33m'+"The Group Label Titles are "+'\033[0m', df_temp.title.to_list())
    plt.show()
for IMG_IDX in [1,2]: 
    temp_img = cv2.imread(TRAIN_BASE+train['image'].values[IMG_IDX*1000])
    dims = np.shape(temp_img)
    ct = np.reshape(temp_img, (dims[0] * dims[1], dims[2]))

    print(dims, " => ", ct.shape)
    plt.title(train['label_group'].values[IMG_IDX*1000])
    sns.distplot(ct[:,0], bins=15)
    sns.distplot(ct[:,1], bins=15)
    sns.distplot(ct[:,2], bins=15)
    plt.show()
"""
### Learning through and about NLP
#### Label Distribution
"""
def plot_hist(col, xlabel, ylabel, title):
    ax = plt.subplots(figsize =(10, 5));
    ax = sns.distplot(temp_nlp[col], kde=True);
    ax.set_ylabel(ylabel, size=15)
    ax.set_xlabel(xlabel, size=15)
    ax.set_title(title, size=20)
    plt.show()
    
temp_nlp = train.copy()
temp_nlp['title_len'] = temp_nlp['title'].apply(lambda x: len(x))
temp_nlp['title_word_count'] =temp_nlp["title"].apply(lambda x: len(str(x).split(" ")))
temp_nlp['title_char_count'] = temp_nlp["title"].apply(lambda x: sum(len(word) for word in str(x).split(" ")))
temp_nlp['title_avg_word_length'] = temp_nlp['title_char_count'] / temp_nlp['title_word_count']
groups = temp_nlp.label_group.value_counts()

plt.figure(figsize=(20,5))
plt.plot(np.arange(len(groups)),groups.values)
plt.ylabel('Duplicate Count',size=14)
plt.xlabel('Index of Unique Item',size=14)
plt.title('Duplicate Count vs. Unique Item Count',size=16)
plt.show()
wordcloud = WordCloud(background_color='white', stopwords=STOPWORDS, width=2560, height=1440).generate(' '.join(train['title']))

ax = plt.subplots(figsize=(15, 15), facecolor='w')
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.tight_layout(pad=0)
for col,x,y,title in [('title_len', 'Length Of Title', 'Num Observations', 'Distribution of Title Length'),
          ('title_word_count', 'Num Words', 'Num Observations', 'Distribution of Word Count for Title'),
          ('title_char_count', 'Num Characters', 'Num Observations', 'Distribution of Characters for Title'),
          ('title_avg_word_length', 'Avg Word Length', 'Num Observations', 'Distribution of Avg Word Length for Title')]:
    plot_hist(col,x,y,title)
del temp_nlp
"""
## Lets Just Build the model
"""
PREDICT_SCORE = True
TRAINING = True
if len(test) > 3:
    TRAIN_BASE = TEST_BASE
    ## Right now i am creating a dummy label column for test data later on i will predict this column as well
    dummy_label_group = list(train.label_group.values)*3
    test['label_group'] = dummy_label_group[:test.shape[0]].copy()
    train = test
    PREDICT_SCORE = False
    TRAINING = False
del test
class DataGenerator(K.utils.Sequence):
    def __init__(self, df, batchSize, code_to_labels, filepath = TRAIN_BASE, shuffle = False, 
                 img_size = IMG_SIZE, classes = NUM_CLASSES):
        self.df = df
        self.shuffle = shuffle
        self.indexes = np.arange(len(df))
        self.path = filepath
        self.batch = batchSize 
        self.img_size = IMG_SIZE
        self.label_dict = code_to_labels
        self.n_classes = classes
        
    def __len__(self):
        '''Total number of steps in a epoch'''
        return int(np.floor(self.df.shape[0]/self.batch))
    
    def on_epoch_end(self):
        if self.shuffle:
            np.random.shuffle(self.indexes)
            
    def __getitem__(self, index):
        '''Generate One batch of files'''
        indexes = self.indexes[index*self.batch:(index+1)*self.batch]
        temp_df = self.df.iloc[indexes]
        X = np.zeros(((len(indexes), self.img_size, self.img_size, 3)))
        Y = np.zeros((self.batch, 1))
        for idx,(index, row) in enumerate(temp_df.iterrows()):
            img = cv2.imread(self.path + row.image)
            X[idx,] = cv2.resize(img, (self.img_size, self.img_size)) / 255
            Y[idx,] = self.label_dict[row.label_group]
        return X, tf.keras.utils.to_categorical(Y, num_classes=self.n_classes)

def f1_score(y_true, y_pred):
    y_true = y_true.apply(lambda x: set(x.split()))
    y_pred = y_pred.apply(lambda x: set(x.split()))
    intersection = np.array([len(x[0] & x[1]) for x in zip(y_true, y_pred)])
    len_y_pred = y_pred.apply(lambda x: len(x)).values
    len_y_true = y_true.apply(lambda x: len(x)).values
    f1 = 2 * intersection / (len_y_pred + len_y_true)
    return f1

def clean(text):
    text = ''.join([k for k in text if k not in string.punctuation])
    text = str(text).lower()
    text = re.sub('[^a-zA-Z]', ' ', text)
    text = re.sub(' +', ' ', text)
    emoji_pattern = re.compile("["
                               u"\U0001F600-\U0001F64F"  
                               u"\U0001F300-\U0001F5FF"  
                               u"\U0001F680-\U0001F6FF"  
                               u"\U0001F1E0-\U0001F1FF"  
                               "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    return text


def train_test_split_data(df, features, label, test_size = 0.33):
    train_x, val_x, train_y, val_y = train_test_split(df[features], df[label], test_size = test_size,
                                                      random_state = SEED, shuffle = True, stratify = df[label])
    return train_x, val_x, train_y, val_y

def learning_rate_scheduler():
    starting_pt_lr   = 0.0001
    exp_decay = 0.1
    def lrfn(epoch):
        if epoch < 5:
            return starting_pt_lr
        else:
            return starting_pt_lr * math.exp(-exp_decay * epoch)
    lr = K.callbacks.LearningRateScheduler(lrfn, verbose = True)
    return lr

tqdm.pandas()
train['title'] = train['title'].progress_apply(clean)
train_x, val_x, train_y, val_y = train_test_split_data(train, ['image'], ['label_group'])
train_x['label_group'] = train_y
train_x.reset_index(inplace = True, drop = True)
val_x['label_group'] = val_y
val_x.reset_index(inplace = True, drop = True)

params = {'batchSize': 5,
          'code_to_labels': encoded_to_label,
          'filepath': TRAIN_BASE,
          'shuffle': False,
          'img_size' : IMG_SIZE,
          'classes' : NUM_CLASSES}

training_generator = DataGenerator(train_x, **params)
validation_generator = DataGenerator(val_x, **params)
complete_generator = DataGenerator(train[['image', 'label_group']], **params)

gc.collect()
def get_multi_pretrained_model(pretrained_layer1, 
                               pretrained_layer2, 
                               classes = NUM_CLASSES):
    
    inp = L.Input(shape = (IMG_SIZE, IMG_SIZE, 3))
    
    ## PretrainedLayer 1
    a = pretrained_layer1(inp)
    a = L.GlobalAveragePooling2D()(a)
    a = L.Dense(750, activation = 'relu')(a)
    a = L.BatchNormalization()(a)
    
    ## PretrainedLayer2
    b = pretrained_layer2(inp)
    b = L.GlobalAveragePooling2D()(b)
    b = L.Dense(750, activation = 'relu')(b)
    b = L.BatchNormalization()(b)
    
    x1 = L.concatenate([a,b])
    x = L.Dense(2048, activation = 'relu')(x1)
    out = L.Dense(classes, activation = 'softmax')(x)

    model = K.models.Model(inputs = inp, 
                           outputs = out)
    image_embeddings = K.models.Model(inputs = inp, 
                           outputs = x1)
    opt = K.optimizers.Adam(learning_rate = 0.0001)

    model.compile(loss='categorical_crossentropy', 
                  optimizer=opt, 
                  metrics=['accuracy'])
    return model, image_embeddings

pretrained_layer1 = K.applications.EfficientNetB0(include_top=False, 
                                                  weights='imagenet', 
                                                  input_shape=(IMG_SIZE, IMG_SIZE, 3))
pretrained_layer2 = K.applications.InceptionV3(include_top=False, 
                                               weights='imagenet', 
                                               input_shape=(IMG_SIZE, IMG_SIZE, 3))

model_pretranied, image_embeddings = get_multi_pretrained_model(pretrained_layer1, 
                                                                pretrained_layer2)

image_embeddings.summary()
if TRAINING: 
    checkpt = tf.keras.callbacks.ModelCheckpoint(f'model_weights_{SEED}.h5', 
                                                 monitor='val_loss', 
                                                 verbose=1, 
                                                 save_best_only=True,
                                                 save_weights_only=True, 
                                                 mode='min')
    checkpointeffnet = tf.keras.callbacks.ModelCheckpoint('best_model_eff_net.h5', 
                                                          monitor='val_loss', mode='min', save_best_only=True,verbose=1)
    # Train model on dataset
    model_pretranied.fit_generator(generator = training_generator, 
                        epochs = EPOCHS,
                        validation_data=validation_generator,
                        callbacks = [learning_rate_scheduler(), checkpt])
else:
    model.load_weights(f'competition_data/model_weights_{SEED}.h5')
    
image_embeddings = image_embeddings.predict(complete_generator, 
                                batch_size=10, 
                                verbose = 0)
"Image embedding shape is :- ", image_embeddings.shape
x_tr = train['title']
y_tr = train['label_group']
tokenizer = Tokenizer()

tokenizer.fit_on_texts(list(x_tr))

x_tr_seq  = tokenizer.texts_to_sequences(x_tr) 
x_tr_seq  = pad_sequences(x_tr_seq, maxlen=50)

size_of_vocabulary=len(tokenizer.word_index) + 1 # +1 for padding
print(size_of_vocabulary)
embeddings_index = {}
with open("../input/pretrainedfiles/model.txt", encoding='utf-8', errors='ignore') as f:
    for line in f:
        word, coefs = line.split(maxsplit=1)
        coefs = np.fromstring(coefs, "f", sep=" ")
        embeddings_index[word] = coefs

print("Found %s word vectors." % len(embeddings_index))
embedding_matrix = np.zeros((size_of_vocabulary, 100))

for word, i in tokenizer.word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        embedding_matrix[i] = embedding_vector
        
del embeddings_index
def get_text_model():
    inp = K.layers.Input(shape=(50,))
    embed = K.layers.Embedding(size_of_vocabulary,100,weights=[embedding_matrix],trainable=True)(inp)
    lstm = K.layers.LSTM(16,return_sequences=True,dropout=0.2)(embed)
    pool = K.layers.GlobalMaxPooling1D()(lstm)

    dense1 = K.layers.Dense(512,activation='relu')(pool)
    out = K.layers.Dense(NUM_CLASSES,activation='sigmoid')(dense1)
    
    model = K.models.Model(inputs = inp, outputs = out)
    embedding_model = K.models.Model(inputs = inp, outputs = pool)
    #Add loss function, metrics, optimizer
    model.compile(optimizer='adam', loss='binary_crossentropy',metrics=["acc"]) 
    return model, embedding_model

earlystopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1,patience=3)  
checkpoint = tf.keras.callbacks.ModelCheckpoint('best_model.h5', monitor='val_acc', mode='max', save_best_only=True,verbose=1)

model_text, embedding_model = get_text_model()

# embedding_model.summary()
# model_text.fit(np.array(x_tr_seq),tf.keras.utils.to_categorical(y_tr),
#                     batch_size=2,epochs=2,
# #                     validation_data=(np.array(x_val_seq),np.array(y_val)),
#                     verbose=1,callbacks=[earlystopping, checkpoint])
def get_text_predictions(df, max_features = 25_000):
    
    model = TfidfVectorizer(stop_words = 'english', binary = True, max_features = max_features)
    text_embeddings = model.fit_transform(df_cu['title']).toarray()
    preds = []
    CHUNK = 1024*4

    print('Finding similar titles...')
    CTS = len(df)//CHUNK
    if len(df)%CHUNK!=0: CTS += 1
    for j in range( CTS ):

        a = j*CHUNK
        b = (j+1)*CHUNK
        b = min(b,len(df))
        print('chunk',a,'to',b)

        cts = cupy.matmul( text_embeddings, text_embeddings[a:b].T).T

        for k in range(b-a):
            IDX = cupy.where(cts[k,]>0.75)[0]
            o = df.iloc[cupy.asnumpy(IDX)].posting_id.values
            preds.append(" ".join(o))
    
    del model,text_embeddings
    gc.collect()
    return preds

df_cu = cudf.DataFrame(train)
text_preds = get_text_predictions(train)
def combine_preds(x):
    all_combined = x['image_preds']+" "+ x['text_preds']
    return ' '.join( set(all_combined.split(" ")) )

def get_nearest_neighors(df, embeds, n = 50, image = True, predict_score = PREDICT_SCORE):
    model = NearestNeighbors(n_neighbors = n)
    model.fit(embeds)
    dist_arr, idx_arr = model.kneighbors(embeds)
    if predict_score:
        scores = []
        if image:
            print("Predicting for image")
            scores = np.arange(0,10.0,0.5)
        else:
            print("Predicting for text")
            scores = np.arange(20,35,0.5)
            
        allscores = []
        for th in scores:
            preds = []
            for idx in range(embeds.shape[0]):
                index_clear_of_th = np.where(dist_arr[idx,] < th)[0]
                preds.append(' '.join(df.posting_id.iloc[idx_arr[idx,index_clear_of_th]].values))
            
            df["pred_values"] = preds
            df["f1_score"] = f1_score(df['matches'], df['pred_values'])
            print(f"f1 Score for the threshold {th} is {df['f1_score'].mean()}")
            allscores.append(df['f1_score'].mean())
        
        score_df = pd.DataFrame({"All_Scores" : allscores, "Thresholds" : scores})
        best_record = score_df[score_df.All_Scores == score_df.All_Scores.max()]
        print(f"Best iteration is with score {best_record.All_Scores.values} and threshold {best_record.Thresholds.values}")
        
        preds = []
        th = best_record.Thresholds.values[0]
            
        for idx in range(embeds.shape[0]):
            index_clear_of_th = np.where(dist_arr[idx,] < th)[0]
            preds.append(" ".join(df.posting_id.iloc[idx_arr[idx,index_clear_of_th]].values))
            
    else:
        preds = []
        th = 0
        if image:
            print("Predicting for image")
            th = 2.4
        else:
            print("Predicting for text")
            th = 24.0
            
        for idx in range(embeds.shape[0]):
            index_clear_of_th = np.where(dist_arr[idx,] < th)[0]
            preds.append(" ",join(df.posting_id.iloc[idx_arr[idx,index_clear_of_th]].values))
            
    return df, preds

if PREDICT_SCORE:
    tmp = train.groupby(['label_group'])['posting_id'].unique().to_dict()
    train['matches'] = train['label_group'].map(tmp)
    train['matches'] = train['matches'].apply(lambda x: ' '.join(x))
    
train, image_preds = get_nearest_neighors(train, image_embeddings, n = 50, image = True)
# train, text_preds = get_nearest_neighors(train, text_embeddings, n = 50, image = False)

train['image_preds'] = image_preds
train['text_preds'] = text_preds
train['matches'] = train.apply(combine_preds, axis = 1)
train[['posting_id', 'matches']].to_csv('submission.csv', index = False)