# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: 'Python 3.8.11 64-bit (''nlp'': conda)'
#     name: python3
# ---

# +
a = [[10, 20], [30, 40], [50, 60]]

for i in range(len(a)):            # 세로 크기
    for j in range(len(a[i])):     # 가로 크기 a[i] [값, 값]의 len은 2
        print(a[i][j], end=' ')
    print()

# +
a = [[10, 20], [30, 40], [50, 60]]

for i in a:
    for j in i:
        print(j)          # 인덱스를 1 증가시킴

# +
import pandas as pd

data = pd.read_csv('한국경제 크롤링.csv', encoding = 'cp949')
# -

text = data.body
# text

# +
# 품사 태깅

from konlpy.tag import Okt  

tagged_words = []
okt=Okt()
for line in text:
    tagged_words.append(okt.pos(line))

print(tagged_words[0])

# from konlpy.tag import Kkma  
# kkma=Kkma()  
# for line in text:
#     print(kkma.pos(line))

# +
# 빈도수세기 word frequency counter

from collections import Counter

count = []
for k in tokens:
    count.append(Counter(k))
print(count[:2], end=" ")


# word_list = count.most_common(10)
# for v in word_list:
#     print(v)

# +
# foreign, josa, punctuation 제거

pos = ["Foreign", "Josa", "Punctuation", "URL"]
clean_words = []
for lt in tagged_words:
    for tags in lt:
        if tags[1] not in pos:
            clean_words.append(tags[0])

# clean_words
print(clean_words[:20])

# +
punct = "/-'?!.,#$%\'()*+-/:;<=>@[\\]^_`{|}~" + '""“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\×™√²—–&'
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }

def clean_punc(text, punct, mapping): 
    for p in mapping: 
        text = text.replace(p, mapping[p])

    for p in punct: 
        text = text.replace(p, f' {p} ')

    specials = {'\u200b': ' ', '…': ' ... ', '\ufeff': '', 'करना': '', 'है': ''} 
    for s in specials: 
        text = text.replace(s, specials[s])

    return text.strip()

no_punc = []
for line in text:
    no_punc.append(clean_punc(line, punct, punct_mapping))
no_punc[0]
# -

# ! git clone https://github.com/SOMJANG/Mecab-ko-for-Google-Colab.git
# ! bash Mecab-ko-for-Google-Colab/install_mecab-ko_on_colab190912.sh

# +
from eunjeon import Mecab

tokenizer= Mecab()

tokens = []
for i in no_punc:
    tokens.append(tokenizer.morphs(i))
print(tokens[0], end=' ')
# -



StarcoderdataJupyterScriptsDedupFiltered