# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

from gensim.models import Word2Vec
w2v = Word2Vec.load("model.pkl")

# +
import numpy as np

word = 'горшок'

similar = w2v.most_similar(positive=[word], topn=100)
for w, wt in similar[:10]:
    print(w, wt)
    
words = np.array([w for w, _ in similar])
word_vectors = np.array([w2v[w] for w in words])
# -

import kmeans 
n_senses = 6
km = kmeans.KMeans(word_vectors, k=n_senses, metric='cosine')

for sense in range(n_senses):
    #sense_words = [w for w, _sense in zip(words, clf_res) if _sense == sense]
    sense_words = list(words[km.Xtocentre == sense])
    sense_words.sort(key=lambda w: w2v.vocab[w].count, reverse=True)
    print(sense, ' '.join(sense_words[:5]))

# +
from sklearn.cluster import KMeans

n_senses = 6
clf = KMeans(n_clusters=n_senses, random_state=42)
clf_res = clf.fit_predict(word_vectors)

from collections import Counter
Counter(clf_res)

# +
from sklearn.decomposition import PCA

pca = PCA(n_components=2)
pca_res = pca.fit_transform(word_vectors)

print(pca_res[:2])

# +
# %matplotlib inline

import pandas as pd

df = pd.DataFrame.from_dict({
    "x": pca_res[:,0],
    "y": pca_res[:,1],
    "clf": clf_res,
})
df.plot.scatter(x="x", y="y", c="clf")

StarcoderdataJupyterScriptsDedupFiltered