# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/ginttone/multi_deeplearning/blob/master/NLP/countvectorizer.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="yOuZoRwusbo3"
# # 뉴스 관련단어 추천 서비스

# + [markdown] id="K7izDVgyCxUb"
# ## 데이터 가져오기

# + id="x935a969BGrO"
import pandas as pd

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="Hdbk_hK8BkFC" outputId="59c70c6e-68c0-49fc-96b8-45085350bdbc"
df = pd.read_csv('./drive/MyDrive/Multi_test_deeplearning/smtph_total.csv')
df.head(5)

# + colab={"base_uri": "https://localhost:8080/"} id="iMaYAQRYCHVC" outputId="766050d2-4200-4225-8a63-979a1e800b72"
df.columns

# + colab={"base_uri": "https://localhost:8080/"} id="Stu5wgLXBxxj" outputId="b7e690b0-2818-4732-b096-6e6df766ef07"
posts = df['Title']+['Description']
type(posts),posts

# + [markdown] id="7GXLfc3RCpJz"
# ## 단어로 쪼개기

# + id="APJhqSoTCO0-"
# !python3 -m pip install konlpy

# + id="-V-7-AMZCjCR"
from konlpy.tag import Okt

# + id="0oq_nQ4kDM8o"
okt = Okt()

# + id="TVrAfOr6EQdz"
stop_words = ['강,', '글,', '애', '미', '번', '은', '이', '것', '등', '더', '를', '좀', '즉', '인', '옹', '때', '만', '원', '이때', '개']

# + colab={"base_uri": "https://localhost:8080/"} id="0n6NBz9SDQYF" outputId="5ef85155-ea7c-4a3e-9dde-5efc6d06e5b5"
posts_noun = []
for post in posts:
  #print(okt.nouns(post))
  for noun in okt.nouns(post):
    posts_noun.append(noun)

len(posts_noun)

# + id="vfkvejxsWP3v"
posts_noun

# + id="FmzXWEsQD9wX"
from collections import Counter

# + [markdown] id="PupbAFjIYW-E"
# type(noun_cnt) 딕셔너리로 출력 됨

# + colab={"base_uri": "https://localhost:8080/"} id="885Bv5vJWPX2" outputId="1d45604e-f329-464e-83cb-8650411276d1"
noun_cnt = Counter(posts_noun)
type(noun_cnt)

# + id="6wxKwmjgWgB2" colab={"base_uri": "https://localhost:8080/"} outputId="0e9c6ff9-5059-44d5-c415-a0054a145664"
top_30_nouns = noun_cnt.most_common(30)
type(top_30_nouns),top_30_nouns

# + colab={"base_uri": "https://localhost:8080/"} id="XiDj6D4QY9Ml" outputId="179b5f09-6ddd-41e6-83dd-4c90c4427ef1"
top_nouns_dict = dict(top_30_nouns)
type(top_nouns_dict)

# + [markdown] id="Gcvc427UZJAM"
# ## Word cloud
#

# + id="DKTk6cSyZBJV"
from wordcloud import WordCloud

# + id="05JBkyc5cIO2"
path = ''

# + id="991uzYxbZQ6n"
nouns_wordcloud = WordCloud()

# + colab={"base_uri": "https://localhost:8080/"} id="Tq83WVJiZan1" outputId="22653b85-b8e2-4916-e3d4-15ac95285bc1"
nouns_wordcloud.generate_from_frequencies(top_nouns_dict)

# + [markdown] id="pnAaoTM4Z_2d"
# ## Display

# + id="zgcPWY2zZwiG"
import matplotlib.pyplot as plt

# + colab={"base_uri": "https://localhost:8080/", "height": 237} id="Xu0XyRcTZ500" outputId="0dfb3bf9-756d-4dcc-e299-22257aff9be0"
plt.imshow(nouns_wordcloud)

# + [markdown] id="glYmo-OEcuuU"
# stopwords, fonts 사용 안한 것

# + [markdown] id="nMsUnNdocO0U"
# ## CountVectorizer(빈도수 사용)

# + id="9qwxE3mVaOLz"
from sklearn.feature_extraction.text import CountVectorizer 

# + id="UWy-7_vifafU"
corpus = [
    'you know I want your love',
    'I like you',
    'what should I do ',    
]

# + id="FwHR0IB5fdXl"
countvectorizer  = CountVectorizer()

# + colab={"base_uri": "https://localhost:8080/"} id="QR5a5OmPfp4D" outputId="7ff4e4b7-7d46-456a-e66b-1f1378ab43a4"
countvectorizer.fit_transform(corpus).toarray()

# + [markdown] id="1lJ2HgszgBGM"
# vocabulary_ 로 컬럼의 순서 확인

# + colab={"base_uri": "https://localhost:8080/"} id="5xCLPghDf1tC" outputId="8ac02fa1-6779-4c04-b077-0ccc8ceac4f7"
print(countvectorizer.vocabulary_)

# + [markdown] id="Yj2Fgg1hV4tj"
# ## Word2Vec
# 리스트 라인의 단어를 백터화 시키는것

# + colab={"base_uri": "https://localhost:8080/"} id="ls2TKGGh7dU-" outputId="a13397fc-68a6-45ab-a4f6-7f6b425844c4"
posts_noun[0:10]

# + id="86q_RiyVV7_z"
from gensim.models import Word2Vec

# + id="1aepym8AWBxs"
word2vec= Word2Vec([posts_noun],min_count=1)

# + colab={"base_uri": "https://localhost:8080/"} id="GuBLiWTwWTR2" outputId="0e8ab727-e260-43ed-8175-6687c7764606"
word2vec

# + [markdown] id="Ebu7m55mWXzI"
# 궁금한 단어 넣으면 유사단어 가져옴(코사인 기준)

# + colab={"base_uri": "https://localhost:8080/"} id="ytKrObm9WWFv" outputId="6c7ac14d-9a04-4019-cc6a-f193c72cef09"
word2vec.wv.most_similar('갤럭시')

# + id="e9WKeYCtWlAb" colab={"base_uri": "https://localhost:8080/"} outputId="0cbb7383-4155-414e-d0d7-3e3394842f4d"
word2vec

# + id="85KAolJM74Gc"


StarcoderdataJupyterScriptsDedupFiltered