# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
from collections import Counter
from glob import glob
import csv
import os.path as path

from pie_extended.cli.utils import get_tagger
from pie_extended.models.dum.imports import get_iterator_and_processor

path_name = 'txt/*.txt'

# +
#_____________ PIE-EXTENDED  _________________
# -

model_name = "dum"
tagger = get_tagger(model_name, batch_size=256, device="cpu", model_path=None)


def lemmatization_pie_extended(path_name, doc_name):
    with open(path_name, encoding="utf8") as file:
        lignes = file.readlines()


        for ligne in lignes:
            iterator, processor = get_iterator_and_processor()
            annotation = tagger.tag_str(ligne, iterator=iterator, processor=processor)
        
    keys = annotation[0].keys()

    with open('csv_textes_annotes_pie/'+doc_name+'.csv', 'w', newline='', encoding="utf-8") as output_file:
        dict_writer = csv.DictWriter(output_file, keys)
        dict_writer.writeheader()
        dict_writer.writerows(annotation)


def moulinette(path_name):
    for doc in glob(path_name):
        doc_name = path.splitext(path.basename(doc))[0]
        print(doc_name)
        lemmatization_pie_extended(doc, doc_name)


moulinette(path_name)



# +
#_____________ TABLE FREQ LEMMA && POS _________________
# -

path_name = 'csv_textes_annotes_pie/*.csv'


def table_freq(path_name, doc_name):
    csv_courant = pd.read_csv('csv_textes_annotes_pie/'+doc_name+'.csv')
    
    list_lemma = csv_courant['lemma'].values.tolist()
    table_freq_lemma = dict(Counter(list_lemma))
    
    list_pos = csv_courant['pos'].values.tolist()
    table_freq_pos = dict(Counter(list_pos))
    
    
    zipped_lemma = list(zip(table_freq_lemma.keys(), table_freq_lemma.values()))
    df_lemma = pd.DataFrame(zipped_lemma, columns=['Lemma', 'Nombre d\'apparitions'])
    df_lemma['Frequence d\'apparition'] = round(df_lemma['Nombre d\'apparitions']/sum(table_freq_lemma.values()),5)
    
    df_lemma.sort_values('Frequence d\'apparition',ascending=False).to_csv(r'tables_des_frequences_lemma/'+doc_name+'_lemma_freq.csv', index = False)

    
    zipped_pos = list(zip(table_freq_pos.keys(), table_freq_pos.values()))
    df_pos = pd.DataFrame(zipped_pos, columns=['POS', 'Nombre d\'apparitions'])
    df_pos['Frequence d\'apparition'] = round(df_pos['Nombre d\'apparitions']/sum(table_freq_pos.values()),5)
    
    df_pos.sort_values('Frequence d\'apparition',ascending=False).to_csv(r'tables_des_frequences_pos/'+doc_name+'_pos_freq.csv', index = False)


def moulinette_table_freq(path_name):
    for doc in glob(path_name):
        doc_name = path.splitext(path.basename(doc))[0]
        print(doc_name)
        table_freq(doc, doc_name)


moulinette_table_freq(path_name)



StarcoderdataJupyterScriptsDedupFiltered