# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import os
import sys
print(f'Python version    \t: {sys.version:.>{15}}')
current_dir_path = os.path.dirname(os.path.realpath(os.getcwd()))
print(f'Current working dir\t: {current_dir_path:.>{15}}')


# ### reading an utf-8 text file in a list

# +
book_fr_path = './data/pg17489.txt'
with open(book_fr_path,mode='r',encoding='utf-8') as f:
    all_book = f.readlines()

num_lines_all_book = len(all_book)
print(f'Number of lines of all the book : {num_lines_all_book}')
# -

# ### finding relevant text to extract indices 

# +
import re
indices = [i for i, elem in enumerate(all_book) if re.match(r'\*\*\* (START|END) OF THIS PROJECT GUTENBERG EBOOK', elem)]

print(f'found this lines at index {indices}')
['[{:5}] {}'.format(i,all_book[i]) for i in indices]

# -

# ### let's get the real book content and get rid of the end of line char

real_book = [i.rstrip() for i in all_book[indices[0]+1:indices[1]-1]]
# let's show 10 first lines
real_book[:10]  

# and the 10 last lines
real_book[-10:]  


# let's adjust a litle bit to get rid of the project gutenberg information 
# that's not part of the original V.Hugo book
real_book = real_book[5:-5]
num_lines_real_book = len(real_book)
print(f'Number of lines of all the real book : {num_lines_real_book}')
real_book[:10] 

real_book[-10:] 

# ### let's extract the Table of Contents

indice_table_content = [i for i, elem in enumerate(real_book) if re.match(r'TABLE DES MATIÈRES', elem)][0]
print('[{:5}] {}'.format(indice_table_content,real_book[indice_table_content]))


# ### we will first find the sections indices of the entire book

indices_sections  = [i for i, elem in enumerate(real_book) if re.match(r'^Livre (.+)\-\-(.+)$',elem)]
print(f'found this Section information at index {indices_sections}')
['[{:5}] {}'.format(i,real_book[i]) for i in indices_sections]

# ### we can use the above list indices_sections to extract the first section

section1 = real_book[117:2708]
print(len(section1))
section1[-10:]

# ### let's find the distinct chapters indices in this first section

indices_chapitres  = [i for i, elem in enumerate(section1) if re.match(r'^Chapitre\s([IVX]+)$',elem)]
print(f'found the chapters in this first section at index {indices_sections}')
['[{:5}] {}'.format(i,section1[i]) for i in indices_chapitres]

# ### now let's grab the first chapter of the first section

section1_chapter_01 = section1[5+1:122-1]
section1_chapter_01


# ## Time to generalisation...
# now it's time to generalize above, and to make it reusable for future exploration of nlp

# +
def get_lines_from_file(file_path, file_encoding='utf-8'):
    """reads a text file at file_path, with specified file_encoding 
    and returns a list of the lines (array of lines) found in this files"""
    with open(file_path,mode='r',encoding=file_encoding) as f:
        return f.readlines()

def get_indices_for_regex(lines_array, regex_expression):
    """ returns a list of the indices in the line_array list of strings, 
    that match the given regex_expression
    """
    return [i for i, elem in enumerate(lines_array) if re.match(regex_expression, elem)]

def print_indices_text(lines_array, the_indexes, message=''):
    """will print the lines from the given lines_array at all the_indexes parameter """
    print(f'found this {message} at indexes {the_indexes}')
    print("\n".join(['[{:5}] {}'.format(i,lines_array[i]) for i in the_indexes]))
    
def get_real_book_from_gutenberg_file(gutenberg_file_path, file_encoding='utf-8', offset=1):
    a_book = get_lines_from_file(gutenberg_file_path)
    gutenberg_indices = get_indices_for_regex(a_book,r'\*\*\* (START|END) OF THIS PROJECT GUTENBERG EBOOK')
    if len(gutenberg_indices) == 2:
        my_real_book = [i.rstrip() for i in a_book[gutenberg_indices[0]+1:gutenberg_indices[1]-1]]
        return my_real_book[offset:-offset] 

def print_head_and_tail(lines_array, offset=10):
    print("### 10 FIRST LINES : ###\n{}\n########################".format("\n".join(lines_array[:offset])))
    print("### 10 LAST  LINES : ###\n{}\n########################".format("\n".join(lines_array[-offset:])))



# -

{elem:  for i, elem in enumerate(real_book) if re.match(r'^Livre (.+)\-\-(.+)$', elem)}

# +
####  let's verify that all the above functions are working as expected
my_big_book = get_lines_from_file(book_fr_path)
num_lines_my_big_book = len(my_big_book)
if num_lines_all_book != num_lines_my_big_book :
    print(f"""
    WARNING[Problem in function get_lines_from_file] :
    in the original books ther is \t {num_lines_all_book} lines, 
    with get_lines_from_files i get\t {num_lines_my_big_book} lines 
    """)
else:
    print(f'OK : found {num_lines_my_big_book} lines in the file {book_fr_path}')

my_indices = get_indices_for_regex(my_big_book,r'\*\*\* (START|END) OF THIS PROJECT GUTENBERG EBOOK')

if my_indices != indices:
    print(f"""
    WARNING[Problem in function get_indices_for_regex] :
    in the original books the indices where \t {indices} , 
    with get_indices_for_regex i get\t {my_indices} instead ! Check the function ! 
    """)
else:
    print_indices_text(my_big_book, my_indices)
    
my_gutenberg_book = get_real_book_from_gutenberg_file(book_fr_path, file_encoding='utf-8', offset=5)
num_lines_my_gutenberg_book = len(my_gutenberg_book)

if num_lines_real_book != num_lines_my_gutenberg_book:
    print(f"""
    WARNING[Problem in function my_gutenberg_book] :
    in the original 'real' book there is \t {num_lines_real_book} lines, 
    with get_lines_from_files i get\t {num_lines_my_gutenberg_book} lines instead ! Check the function.
    """)
else:
    print(f'OK : found {num_lines_real_book} real lines in the gutenberg file {book_fr_path}')
    
print("### 10 FIRST LINES : ###\n{}".format("\n".join(my_gutenberg_book[:10])))
print("### 10 LAST  LINES : ###\n{}".format("\n".join(my_gutenberg_book[-10:])))
    
# -

my_sections_indices  = get_indices_for_regex(my_gutenberg_book, r'^Livre (.+)\-\-(.+)$')
print(f'found this Section information at index {my_sections_indices}')
['my_gutenberg_book[{:5}]= {}'.format(i,my_gutenberg_book[i]) for i in my_sections_indices]

# let's keep only the book lines of the real sections
my_gutenberg_book = my_gutenberg_book[117:]

print_head_and_tail(my_gutenberg_book)

my_sections_indices  = get_indices_for_regex(my_gutenberg_book, r'^Livre (.+)\-\-(.+)$')
print(f'found this Section information at index {my_sections_indices}')
for i,v in enumerate(my_sections_indices):
    print(f'{i:<{5}}:my_gutenberg_book[{v:>{5}}] = {my_gutenberg_book[v]}')
['my_gutenberg_book[{:5}] = {}'.format(i,my_gutenberg_book[i]) for i in my_sections_indices]

# +
my_sections = {}
for i,v in enumerate(my_sections_indices):
    print(f'{i:<{5}}:{v:>{15}}, {my_gutenberg_book[v]}')
    key = f'{i+1:>{2}}) {my_gutenberg_book[v]}'
    print(key, i < (len(my_sections_indices)-1))
    if i < (len(my_sections_indices)-1):
        my_sections[key]=my_gutenberg_book[v:my_sections_indices[i+1]]
    else:
        my_sections[key]=my_gutenberg_book[v:]    


my_sections.keys()
# -

print_head_and_tail(my_sections[' 1) Livre premier--Un juste'])

print_head_and_tail(section1)

section1 == my_sections[' 1) Livre premier--Un juste']



StarcoderdataJupyterScriptsDedupFiltered