# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="AW0RxUlxYIda" colab={"base_uri": "https://localhost:8080/"} outputId="a4d55390-13f4-48bf-f8e8-4d1892103b43"
import pandas as pd
import io
import re
from os import listdir
import io
import random
import json
import datetime
import argparse
import sys
# #!wget https://raw.githubusercontent.com/danielinux7/Multilingual-Parallel-Corpus/master/ab-ru/quran.tsv

# + colab={"base_uri": "https://localhost:8080/"} id="0YObtr_fI8lP" outputId="10137209-acd2-4ba2-dcce-f44fc1cfd65e"
alphabet_ab = re.compile('[ҟцукенгшәзхҿфывапролджҽџчсмитьбҩҵқӷӡҳԥҷҭ\.\:,;\ 0-9-\(\)"!?]+',re.I)
alphabet_ru = re.compile('[ёйцукенгшщзхъфывапролджэячсмитьбю\.\:,;\ 0-9-\(\)"!?]+',re.I)
alphabe_ab = re.compile('ҟцукенгшәзхҿфывапролджҽџчсмитьбҩҵқӷӡҳԥҷҭ',re.I)
alphabe_ru = re.compile('ёйцукенгшщзхъфывапролджэячсмитьбю',re.I)
sentence_signs = re.compile('[\.\:!?]+',re.I)


#corp=pd.read_csv("/content/quran.tsv", sep='\t')



def mis_punctuation(parallel_corpus):
    filtered_punctuations = 0
    ru_result_list = []
    ab_result_list = []

    for translation in parallel_corpus:
        ru_signs = re.sub(alphabe_ru, "", translation[0]) #replacing alphabe_ru with "" in string translation[0]
        ab_signs = re.sub(alphabe_ab, "", translation[1]) #replacing alphabe_ab with "" in string translation[1]
        if ru_signs == ab_signs: #if both strings have same punctuattion
            ru_result_list.append(translation[0])
            ab_result_list.append(translation[1])
        else:
            filtered_punctuations += 1

    print("filtered punctuations: "+str(filtered_punctuations))

    return list(zip(ru_result_list,ab_result_list))
#corpa=filter_punctuation(corp)

# + id="kK_GSoZEyGFd"


StarcoderdataJupyterScriptsDedupFiltered