# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/cateto/python4NLP/blob/main/colab/Korean_textProcessing.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="BTA6oniQR-GP"
# # PyKoSpacing
#

# + colab={"base_uri": "https://localhost:8080/"} id="WITNZqN3Qv6R" outputId="465453ec-7d1b-473a-e8c3-127114327b38"
# !pip install git+https://github.com/haven-jeon/PyKoSpacing.git

# + id="0QVIFeG3Q15m"
sent = '김철수는 극중 두 인격의 사나이 이광수 역을 맡았다. 철수는 한국 유일의 태권도 전승자를 가리는 결전의 날을 앞두고 10년간 함께 훈련한 사형인 유연재(김광수 분)를 찾으러 속세로 내려온 인물이다.'

# + colab={"base_uri": "https://localhost:8080/"} id="xB8AVBUkReI-" outputId="ebfbab15-3abb-45dd-b3c8-5067b096cfa7"
new_sent = sent.replace(" ", '')
print(new_sent)

# + colab={"base_uri": "https://localhost:8080/"} id="S8cM9fjvRiau" outputId="9cc9a51d-5911-496f-89c6-fea2392e1563"
from pykospacing import spacing

kospacing_sent = spacing(new_sent)
print(sent)
print(kospacing_sent)

# + [markdown] id="ApmQdy30SCM3"
# # Py-Hanspell

# + colab={"base_uri": "https://localhost:8080/"} id="6WXhat4dRtFO" outputId="f3347433-deb3-4fdd-f0a8-d59f933f2c37"
# !pip install git+https://github.com/ssut/py-hanspell.git

# + colab={"base_uri": "https://localhost:8080/"} id="Od4JMp3XSF6O" outputId="c8110e5a-4717-47b1-ee72-c3bc6dfb6294"
from hanspell import spell_checker
sent = "맞춤법 틀리면 외 않되? 쓰고싶은대로쓰면돼지 "
spelled_sent = spell_checker.check(sent)
hanspell_sent = spelled_sent.checked
print(hanspell_sent)

# + colab={"base_uri": "https://localhost:8080/"} id="IjXjiga7SUWm" outputId="1a46cea6-ee70-4cb6-fe92-259b1322d4f2"
spelled_sent = spell_checker.check(new_sent)
hanspell_sent = spelled_sent.checked
print(hanspell_sent)
print(kospacing_sent)

# + [markdown] id="rcDtdQcLS7kn"
# hanspell 결과에 대한 의견 : 띄어쓰기 체크 나쁘지 않음. 근데 왜 등장인물 이름이 바뀐거징? '유연재'가 잘못되었고 '유연제'를 단어로 인식한건가
#

# + colab={"base_uri": "https://localhost:8080/"} id="ChYOXzp-StXV" outputId="0f29bc85-0e76-48dd-cc40-f0508fcd5618"
# !pip install soynlp

# + colab={"base_uri": "https://localhost:8080/"} id="JMDhR0gDUGN1" outputId="a11d0e9a-c1a2-4c48-ca7b-3d0e3796c325"
pip install konlpy

# + colab={"base_uri": "https://localhost:8080/"} id="a0zYpW1oUU73" outputId="e9b19c0e-415a-42e5-a709-325c18c98777"
from konlpy.tag import Okt
tokenizer = Okt()
print(tokenizer.morphs('에이비식스 이대휘 1월 최애돌 기부 요정'))

# + [markdown] id="2BKi-U_YUftV"
# 신조어에 대해 분리된 결과를 나타냄.
# 텍스트 데이터의 빈도수에 따라 형태소를 파악하는 토크나이저 soynlp로 분석한다면?

# + id="y35xxHyyUY81"
import urllib.request
from soynlp import DoublespaceLineCorpus
from soynlp.word import WordExtractor

# + colab={"base_uri": "https://localhost:8080/"} id="x7IUVcebUyY0" outputId="746d5812-0bc6-47c1-f478-99250af4fb58"
urllib.request.urlretrieve("https://raw.githubusercontent.com/lovit/soynlp/master/tutorials/2016-10-20.txt", filename="2016-10-20.txt")

# + colab={"base_uri": "https://localhost:8080/"} id="pBGBl5snU3HF" outputId="98f3fdd8-187a-4e5b-bddc-68738912b47c"
corpus = DoublespaceLineCorpus("2016-10-20.txt")
len(corpus)

# + colab={"base_uri": "https://localhost:8080/"} id="uOd4h6u1U7Lt" outputId="3e8ee100-b48f-4883-ced4-a8ec8aaedb80"
i = 0
for document in corpus:
    if len(document) > 0:
      print(document)
      i = i + 1
    if i == 3:
      break

# + [markdown] id="JbniXmsNVNPK"
# 학습 기반의 단어 토크나이저이기 때문에 기존의 형태소 분석기와는 다르게 학습 과정을 거쳐야 함.
#
# - 응집확률(Cohesion Probability) : 연결되어 자주 등장함
# - 브랜칭 엔트로피 : 주어진 문자열에서 얼마나 다음 문자가 등장할 수 있는지를 판단하는 척도임. 
# 하나의 완성된 단어에 가까워질수록 문맥으로 인해 점점 정확히 예측하면서 브랜칭 엔트로피의 값이 줄어듦.

# + colab={"base_uri": "https://localhost:8080/"} id="GD5vol1BVETF" outputId="4f92450b-1789-481a-fc71-877a5df5c8a5"
word_extractor = WordExtractor()
word_extractor.train(corpus)
word_score_table = word_extractor.extract()

# + colab={"base_uri": "https://localhost:8080/"} id="xLBpymbRV3AE" outputId="0b7896c4-9b4e-469d-aaa7-7064d740998f"
word_score_table["반포한"].cohesion_forward

# + colab={"base_uri": "https://localhost:8080/"} id="vlyW1fpsWPWl" outputId="88a59e91-7bbd-4da5-d285-244c2eea55af"
word_score_table["반포한강"].cohesion_forward

# + colab={"base_uri": "https://localhost:8080/"} id="mJyhuVtoWSVX" outputId="f10e69fc-dce5-45be-c41c-b5b4984dc5e7"
word_score_table["반포한강공"].cohesion_forward

# + colab={"base_uri": "https://localhost:8080/"} id="2TA86pjXWZum" outputId="9fb366b9-bbdc-4d39-add2-f73d0e0c1c4d"
word_score_table["반포한강공원"].cohesion_forward

# + colab={"base_uri": "https://localhost:8080/"} id="sOcRCF16WcxP" outputId="3812b6d2-e721-40b1-9374-b49ecce5a31f"
word_score_table["반포한강공원에"].cohesion_forward

# + [markdown] id="UtiTkSGmo398"
# 브랜칭 엔트로피 : 문자 시퀀스에서 다음 문자 예측을 위해 헷갈리는 정도. (줄어들면 좋겠쥬)

# + colab={"base_uri": "https://localhost:8080/"} id="9_cjl9xRWdpm" outputId="acf3bed2-e5e1-48e4-d150-743c688ca362"
word_score_table["디"].right_branching_entropy

# + colab={"base_uri": "https://localhost:8080/"} id="o2mi6naOpKXw" outputId="a082214e-fff1-4f2b-d132-32c4e284b0cd"
word_score_table["디스"].right_branching_entropy

# + colab={"base_uri": "https://localhost:8080/"} id="8lJFv3ixpM2V" outputId="164131f5-03e6-4fe3-a60b-8978cbf7cbe4"
word_score_table["디스플"].right_branching_entropy

# + colab={"base_uri": "https://localhost:8080/"} id="wRz4KE5JpPFN" outputId="0a5e4669-d960-49bd-e4d3-e13684ce89f3"
word_score_table["디스플레이"].right_branching_entropy

# + colab={"base_uri": "https://localhost:8080/"} id="h4fI_JvtpSDd" outputId="1d5c3e96-0525-42af-ea81-5af845ef81fd"
from soynlp.tokenizer import LTokenizer

scores = { word:score.cohesion_forward for word, score in word_score_table.items()}
l_tokenizer = LTokenizer(scores=scores)
l_tokenizer.tokenize("국제사회와 우리의 노력들로 범죄를 척결하자", flatten=False)

# + colab={"base_uri": "https://localhost:8080/"} id="RH9oQxgJqBO1" outputId="c1d38298-212d-4c92-ad0c-177b6207e4d4"
from soynlp.tokenizer import MaxScoreTokenizer

maxscore_tokenizer = MaxScoreTokenizer(scores=scores)
maxscore_tokenizer.tokenize("국제사회와우리의노력들로범죄를척결하자")

# + colab={"base_uri": "https://localhost:8080/"} id="MSeawbPyqgpD" outputId="3c809823-2e3a-4ab4-8d58-9bbc72b3337a"
from soynlp.normalizer import *
print(emoticon_normalize('앜ㅋㅋㅋㅋ이영화존잼쓰ㅠㅠㅠㅠㅠ', num_repeats=2))
print(emoticon_normalize('앜ㅋㅋㅋㅋㅋㅋㅋㅋㅋ이영화존잼쓰ㅠㅠㅠㅠ', num_repeats=2))

# + colab={"base_uri": "https://localhost:8080/"} id="zPg-QN8Vqq_L" outputId="c27de150-6a19-476d-bd3a-f09b340c55b5"
print(repeat_normalize('와하하하하하하하하하핫', num_repeats=2))

# + colab={"base_uri": "https://localhost:8080/"} id="ER8PwBoiqykj" outputId="ab7295a4-8327-49f8-8f63-2a6396fd8fda"
print(repeat_normalize('와하하하하하하핫', num_repeats=2))

# + colab={"base_uri": "https://localhost:8080/"} id="EcDdsNXwqzur" outputId="8fd446ff-2e5c-422d-9838-d6f6bdccc7e6"
pip install customized_konlpy

# + colab={"base_uri": "https://localhost:8080/"} id="0XVdIWW2rgpF" outputId="9db14f5e-c32d-48b6-a7e8-a13b4f4fb55c"
from ckonlpy.tag import Twitter
twitter = Twitter()
twitter.morphs('은경이는 사무실로 갔습니다.')

# + id="e04-2udprpmT"
twitter.add_dictionary('은경이', 'Noun')

# + colab={"base_uri": "https://localhost:8080/"} id="UDKzpfOHrr3L" outputId="013dec5f-ee3e-4130-c1c8-3ab04109277c"
twitter.morphs('은경이는 사무실로 갔습니다.')

# + id="Wvrrpd90rtX7"


StarcoderdataJupyterScriptsDedupFiltered