# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# !pip install gensim

import nltk
#from gensim.models.phrases import Phraser
from gensim.models import Phrases
import gensim.models.keyedvectors as word2vec

# +
from gensim.models import Phrases
documents=["I am a good boy","<NAME> will be next Prime Minister","AP<NAME> was an Indian scientist"]
sentence_stream = [doc.split(" ") for doc in documents]

#sentence_stream=brown_raw[0:10]
bigram = Phrases(sentence_stream, min_count=1, delimiter=b' ')
trigram = Phrases(bigram[sentence_stream], min_count=1, delimiter=b' ')

for sent in sentence_stream:
    bigrams_ = [b for b in bigram[sent] if b.count(' ') == 1]
    trigrams_ = [t for t in trigram[bigram[sent]] if t.count(' ') == 2]

    print(bigrams_)
    print(trigrams_)
# -

# !wget "https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz"









StarcoderdataJupyterScriptsDedupFiltered