# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="v_r3RBbn79gc"
# # 은전한닢 로드
#
# https://github.com/SOMJANG/Mecab-ko-for-Google-Colab

# + colab={"base_uri": "https://localhost:8080/"} id="f9uvuAYG7vP2" outputId="96059db4-3040-457f-8c75-237bfe537a2a"
# ! git clone https://github.com/SOMJANG/Mecab-ko-for-Google-Colab.git

# + colab={"base_uri": "https://localhost:8080/"} id="FpfUqHb-7vNj" outputId="c05d17db-ebf9-4dfd-d8c4-780db24c159b"
# %cd Mecab-ko-for-Google-Colab

# + id="-cR4xNJk7vLG"
# !bash install_mecab-ko_on_colab190912.sh

# + id="VmG3vDN17vI0"
from konlpy.tag import Mecab

mecab = Mecab()

# + colab={"base_uri": "https://localhost:8080/"} id="Yj6idlCw7vGP" outputId="30622fa0-ccf3-4147-bf40-e2d070c32d86"
mecab.morphs('동해물과 백두산이 마르고 닳도록')

# + [markdown] id="d366dbsVmyQh"
# # 패키지로드

# + id="6W2y29ZMYU7t"
# !pip install transformers
# !pip install tensorboardx

# + [markdown] id="H4RRNgWeGGAd"
# # pyrouge

# + id="Kfmzjd9SnwVk"
# !pip install pyrouge --upgrade
# !pip install https://github.com/bheinzerling/pyrouge/archive/master.zip
# !pip install pyrouge
# !pip show pyrouge
# !git clone https://github.com/andersjo/pyrouge.git
from pyrouge import Rouge155
# !pyrouge_set_rouge_path 'pyrouge/tools/ROUGE-1.5.5'

# + id="_xaaB2uCn-8N"
# !sudo apt-get install libxml-parser-perl

# + id="W9N7cryqoIQU"
# %%shell
# cd pyrouge/tools/ROUGE-1.5.5/data
# rm WordNet-2.0.exc.db # only if exist
# cd WordNet-2.0-Exceptions
# rm WordNet-2.0.exc.db # only if exist

./buildExeptionDB.pl . exc WordNet-2.0.exc.db
# cd ../
ln -s WordNet-2.0-Exceptions/WordNet-2.0.exc.db WordNet-2.0.exc.db

# + [markdown] id="jl7HMxYXFUPf"
# # Load the Repo

# + colab={"base_uri": "https://localhost:8080/"} id="5gYXSrkHFTwr" outputId="0c3a473a-8e1a-4b44-8d67-cc86f5df22d8"
# %cd /content
# !git clone https://github.com/HaloKim/KorBertSum.git
# %cd /content/KorBertSum

# + [markdown] id="6i6OGXdMn8Ba"
# # BERT

# + colab={"base_uri": "https://localhost:8080/", "referenced_widgets": ["<KEY>", "99913821e4214ba9b6a26d307be4a3de", "<KEY>", "<KEY>", "cbc07fa0e01b4c4b90d88d8a254444e0", "4e67e59da09642b3956034426a68f2de", "5faa39d53ed748efaa70c76a29463e35", "ec802a2bcba44f90b2ce829f3ef72a2e", "0536d9a8d2ea49098c4522ae8554b2be", "<KEY>", "74c733cf0408467697091e8f4e182462", "<KEY>", "<KEY>", "<KEY>", "bdbca8ea5cc4480e83c12d7264e940a6", "ecbe9ac4db214cc99cacdc157a6e5e49", "0fbe6dce7bed45bd9439d3c9a616ba31", "<KEY>", "ac14a1e471ad4a8db73a581246d67e4a", "<KEY>", "<KEY>", "99f9c19c3a8a4c23a2212ca21e7bc027", "<KEY>", "2be375bd1a974562b67acda54da92ee3", "<KEY>", "51b02a64c1e44552b2de576ff22cdb92", "<KEY>", "dd65c8725b9e46068e6d5841fec2f896", "16a901e9fb07473ab4dadffaee04d618", "<KEY>", "6f6aa33e782344d983b101682ee580a2", "<KEY>", "4fa80e165e7540e8be6797364dd94440", "c140e9e47adf4e4c8754ce30becd633c", "<KEY>", "bd850beb0eae423a905c54ea80bbec38", "da82776cd0de4bdca3932709c490e92e", "<KEY>", "8edafda4cf7b4af999097bad98e134fb", "9a2df1185dd440b5bb3e1d4123a6ec61"]} id="QWafHU5dbjlK" outputId="4d6b28ae-c009-4b3c-a9fd-0113d776bf73"
from transformers import BertTokenizer, BertModel, BertConfig

config = BertConfig.from_pretrained('bert-base-multilingual-cased')
tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')
model = BertModel.from_pretrained("bert-base-multilingual-cased")

# + [markdown] id="EjHhHI9zm8Y9"
# # Data Package

# + id="6IWz6c4Ifjar"
import json

with open('/content/drive/MyDrive/Colab_Notebooks/Study/DACON/235671_한국어 문서 추출요약 AI 경진대회_data/train.jsonl', 'r') as json_file:
    json_list = list(json_file)
result = []
for json_str in json_list:
    result.append(json.loads(json_str))

# + colab={"base_uri": "https://localhost:8080/", "height": 202} id="bH7IQ9GtiZh1" outputId="775006c9-c5b9-44ff-e1ca-e4f4e517fb1e"
import pandas as pd
df = pd.DataFrame(result)
df.head()

# + colab={"base_uri": "https://localhost:8080/"} id="OeukR_5HrlyH" outputId="775b0352-0986-4ad8-d9b9-893486761571"
test = df
test['morphs'] = 0
for i in range(len(test)):
  arr = []
  for j in test['article_original'][i]:
    arr.append(' '.join(mecab.morphs(j)))
  test['morphs'][i] = arr 

# + id="psok6XNpSPh4"
test.to_csv('/content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/output.csv')

# + [markdown] id="TeFaCj0S8Uml"
# # Load csv

# + colab={"base_uri": "https://localhost:8080/", "height": 285} id="MxkjXDTrSV9M" outputId="8880b39c-1907-4cd7-a7fa-d4ac2b8ed37c"
import pandas as pd
import json

test = pd.read_csv('/content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/output.csv').drop('Unnamed: 0', axis=1)
test.head()

# + id="Y1FWGJcpIxRg"
from sklearn.model_selection import train_test_split

train_set, test_set = train_test_split(test, test_size = 0.2)
valid_set, test_set = train_test_split(test_set, test_size = 0.1)

# + [markdown] id="BNxZvpLWg195"
# # csv to json

# + colab={"base_uri": "https://localhost:8080/"} id="1FhzVU6bODhm" outputId="6d3cb04c-0e75-4934-ed53-6554cc1454eb"
import ast
from tqdm import tqdm
import os

list_dic = []
for idx, row in train_set.iterrows():
  raw = row['morphs']
  target_idx = ast.literal_eval(row['extractive'])

  sentences = raw.split(',')
  src = [i.split(' ') for i in sentences]
  tgt = [a for i,a in enumerate(src) if i in target_idx]
  
  mydict = {}
  mydict['src'] = src
  mydict['tgt'] = tgt
  list_dic.append(mydict)
        
temp = []
for i,a in enumerate(tqdm(list_dic)):
  if (i+1)%6!=0:
      temp.append(a)
  else:
      filename = 'korean.'+'train'+'.'+str(i//6)+'.json'
      with open('/content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master'+'/json/'+filename, "w", encoding='utf-8') as json_file:
          json.dump(temp, json_file, ensure_ascii=False)
      temp = []

# + colab={"base_uri": "https://localhost:8080/"} id="Csh4zkpPJpzu" outputId="ea2e371f-e033-424f-a27a-2b409bfae8d7"
try:
  os.mkdir('/content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/json/val')
except:
  pass

list_dic = []
for idx, row in valid_set.iterrows():
        raw = row['morphs']
        target_idx = ast.literal_eval(row['extractive'])

        sentences = raw.split(',')
        src = [i.split(' ') for i in sentences]
        tgt = [a for i,a in enumerate(src) if i in target_idx]
        
        mydict = {}
        mydict['src'] = src
        mydict['tgt'] = tgt
        list_dic.append(mydict)
        
temp = []
for i,a in enumerate(tqdm(list_dic)):
    
    if (i+1)%6!=0:
        temp.append(a)
    else:
        filename = 'korean.'+'valid'+'.'+str(i//6)+'.json'
        with open('/content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master'+'/json/val/'+filename, "w", encoding='utf-8') as json_file:
            json.dump(temp, json_file, ensure_ascii=False)
        temp = []

# + colab={"base_uri": "https://localhost:8080/"} id="WTE25bv8Jp3A" outputId="80cec4e5-3ebf-451a-d79a-76babd6c1249"
try:
  os.mkdir('/content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/json/test')
except:
  pass

list_dic = []
for idx, row in valid_set.iterrows():
        raw = row['morphs']
        target_idx = ast.literal_eval(row['extractive'])

        sentences = raw.split(',')
        src = [i.split(' ') for i in sentences]
        tgt = [a for i,a in enumerate(src) if i in target_idx]
        
        mydict = {}
        mydict['src'] = src
        mydict['tgt'] = tgt
        list_dic.append(mydict)
        
temp = []
for i,a in enumerate(tqdm(list_dic)):
    
    if (i+1)%6!=0:
        temp.append(a)
    else:
        filename = 'korean.'+'test'+'.'+str(i//6)+'.json'
        with open('/content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master'+'/json/val/'+filename, "w", encoding='utf-8') as json_file:
            json.dump(temp, json_file, ensure_ascii=False)
        temp = []

# + [markdown] id="eD6y73O4fmkY"
# # Json to pt

# + id="tLQfbpzEKS6b"
# 데이터 처리모드, json경로, 저장경로, 로그파일
# !python /content/KorBertSum/src/preprocess.py \
#   -mode format_to_bert \
#   -raw_path /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/json \
#   -save_path /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/bert_data \
#   -dataset train \

# + id="tGiQq9ejKI47"
try:
  os.mkdir('/content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/bert_data/val')
except:
  pass

# !python /content/KorBertSum/src/preprocess.py \
#   -mode format_to_bert \
#   -raw_path /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/json/val \
#   -save_path /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/bert_data/test \
#   -dataset valid \

# + id="xDhVi3KQJ3_c"
try:
  os.mkdir('/content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/bert_data/test')
except:
  pass

# !python /content/KorBertSum/src/preprocess.py \
#   -mode format_to_bert \
#   -raw_path /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/json/test \
#   -save_path /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/bert_data/test \
#   -dataset test \

# + [markdown] id="wa5vcawrGlbo"
# # Train

# + id="nHnvBZO-cwbf"
# !python /content/KorBertSum/src/train.py \
#   -mode train \
#   -encoder classifier \
#   -bert_data_path /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/bert_data/korean \
#   -model_path /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/models/bert_classifier \
#   -log_file /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/logs/train.txt \
#   -lr 2e-3 -visible_gpus 0 -gpu_ranks 0 -world_size 1 -report_every 1000 -dropout 0.5 \
#   -save_checkpoint_steps 1000 -batch_size 3000 -decay_method noam -train_steps 100000 -accum_count 1\ 

# + [markdown] id="cWd-qpB_GoeE"
# # Validation

# + id="7XV4tywmcT_P"
# %cd /content

# !python /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/src/train.py \
#   -mode validate \
#   -bert_data_path /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/bert_data/test/korean \
#   -visible_gpus 0 -gpu_ranks 0 -batch_size 3000 \
#   -log_file /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/logs/val.txt \
#   -temp_dir /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/temp \
#   -result_path /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/results \
#   -test_from /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/models/bert_classifier/model_step_9000.pt \

# + [markdown] id="Xw_x2sIFnGOP"
# # Test

# + id="nisU6_FZEGfx"
# %cd /content

# !python /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/src/train.py \
#   -mode test \
#   -bert_data_path /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/bert_data/test/korean \
#   -visible_gpus 0 -gpu_ranks 0 -batch_size 3000 \
#   -log_file /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/logs/val.txt \
#   -temp_dir /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/temp \
#   -result_path /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/results \
#   -test_from /content/drive/MyDrive/Colab_Notebooks/Study/BertSum-master/models/bert_classifier/model_step_21000.pt \

# + id="0UcXM_1ACwqo"
 

# + id="nGK5O0cGXi_M"


StarcoderdataJupyterScriptsDedupFiltered