# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/cateto/python4NLP/blob/main/colab/Word2Vec_1003_eng.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + colab={"base_uri": "https://localhost:8080/"} id="KHk4ye81sx8e" outputId="4167ba1c-32ce-4cba-d277-0c4cd5832773"
import nltk
nltk.download('punkt')

# + id="VidWOfeGs4Mv"
import urllib.request
import zipfile
from lxml import etree
import re
from nltk.tokenize import word_tokenize, sent_tokenize

# + colab={"base_uri": "https://localhost:8080/"} id="S0FCLZgftCEl" outputId="4a5fad5a-1471-4eb4-db97-e12e7938eb65"
urllib.request.urlretrieve("https://raw.githubusercontent.com/GaoleMeng/RNN-and-FFNN-textClassification/master/ted_en-20160408.xml", filename="ted_en-20160408.xml")

# xml 데이터

# + id="37y2DPyatOlg"
targetXML = open('ted_en-20160408.xml', 'r', encoding='UTF8')
target_text = etree.parse(targetXML)

# xml 파일에서 <conetnt></content> 사이 내용 가져옴
parse_text = '\n'.join(target_text.xpath('//content/text()'))

# + id="yOK4hfvXtmGX"
content_text = re.sub(r'\([^)]*\)', '', parse_text)

# 정규표현식으로 배경음 부분 제거 (audio), (laughter) 즉 괄호로 구성된 내용 제거함

sent_text = sent_tokenize(content_text)

# 입력 코퍼스를 NTLK로 문장 토큰화 수행

normalized_text = []
for string in sent_text:
  tokens = re.sub(r"[^a-z0-9]+", " ", string.lower())
  normalized_text.append(tokens)
# 구두점 제거 및 소문자로 변환

result = [word_tokenize(sentence) for sentence in normalized_text]

# 단어 토큰화 수행 

# # 44초...?

# + colab={"base_uri": "https://localhost:8080/"} id="Q7XjRcXRuSXN" outputId="96f4ef86-db83-4e17-bec7-bda865c6ad70"
print("총 샘플 개수 : {}".format(len(result)))

# + colab={"base_uri": "https://localhost:8080/"} id="3ifNAvmUud-6" outputId="407f84c9-b236-468c-e35a-ffda530e9a6b"
# 샘플 3개만 출력
for line in result[:3]:
    print(line)

# + id="ctLIzO12uh1v"
from gensim.models import Word2Vec
model = Word2Vec(sentences=result, size=100, window=5, min_count=5, workers=4, sg=0)

# + [markdown] id="zVllSvs47z4q"
# - size = 워드 벡터의 특징 값. 즉, 임베딩 된 벡터의 차원.
# - window = 컨텍스트 윈도우 크기
# - min_count = 단어 최소 빈도 수 제한 (빈도가 적은 단어들은 학습하지 않는다.)
# - workers = 학습을 위한 프로세스 수
# - sg = 0은 CBOW, 1은 Skip-gram.

# + colab={"base_uri": "https://localhost:8080/"} id="w1MFq5b97zhl" outputId="e0cb4394-74f2-4736-f0a8-29d53e8ecf39"
model_result = model.wv.most_similar("man")
print(model_result)

# + id="n5VPgu9i8Llu"
from gensim.models import KeyedVectors
model.wv.save_word2vec_format('eng_w2v')

loaded_model = KeyedVectors.load_word2vec_format("eng_w2v")

# + colab={"base_uri": "https://localhost:8080/"} id="SfcLZLwE8dPy" outputId="45956363-f6bb-4888-e70a-8af963f3a21e"
model_result = loaded_model.most_similar("man")
print(model_result)

# + id="kyQGi8wy8nnt"


StarcoderdataJupyterScriptsDedupFiltered