# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python3
#     language: python
#     name: python3
# ---

# cd ../src

# +
from no_ing_model import generate_embeddings

embedding_df, distance_df = generate_embeddings(experiment_mode=True)

# +
import numpy as np

np.array(embedding_df["Embedding"].apply(str).value_counts())

# +
import numpy as np
from sklearn.manifold import TSNE
import seaborn as sns
import matplotlib.pyplot as plt
# %matplotlib inline

COLOR_DIGIT = 2

def plot_embedding(df):
    X_embedded = TSNE(n_components=2, verbose=1).fit_transform(np.array(df["Embedding"].values.tolist()))
    
    code_top_digit = sorted(list(set([code[:COLOR_DIGIT] for code in df["Code"]])))
    
    rgb_values = sns.color_palette("Set2", len(code_top_digit))
    color_map = dict(zip(code_top_digit, rgb_values))
    
    x = X_embedded[:,0]
    y = X_embedded[:,1]

    industry_l = [title for title in df["Title"]]
    color_l = [color_map.get(ind[:COLOR_DIGIT], rgb_values[-1]) for ind in df["Code"]]
    plt.scatter(x,y, c=color_l, linewidths=0)
    
plot_embedding(embedding_df)
# -

embedding_df.sample(10)

distance_df[distance_df["Code_x"] == "111150"].sort_values("pred_dist").head(20)



StarcoderdataJupyterScriptsDedupFiltered