Add of tabula functionality + exception management

reduce_and_convert_PDF.py

@@ -8,6 +8,7 @@ import openpyxl
 from openpyxl.utils.dataframe import dataframe_to_rows
 from openpyxl.styles import numbers
 from openpyxl.worksheet.table import Table, TableStyleInfo
+import tabula
 
 def extract_pages(pdf_path, start_page, end_page, output_path):
     reader = PdfReader(pdf_path)
@@ -42,14 +43,23 @@ def reduce_pdf(pdf_folder,reduced_pdf_folder):
 
 
 
-def extract_tables_camelot(pdf_path):
-    # Extract tables from the PDF file using Camelot
-    tables = camelot.read_pdf(pdf_path, pages='all',flavor='stream')
-    return tables
+
+
+
+def extract_tables_camelot_or_tabula(pdf_path):
+    try:
+        tables = camelot.read_pdf(pdf_path, pages='all', flavor='stream')
+        return [table.df for table in tables]
+    except Exception as e:
+        print(f"Camelot failed with error: {e}")
+        print("Trying with Tabula...")
+        dfs = tabula.read_pdf(pdf_path, pages='all', multiple_tables=True)
+        return dfs
+
 
 def get_numeric_count(row):
     # Get the number of numerical values in a row
-    return sum(1 for x in row if (pd.notna(pd.to_numeric(x.replace(",", "").strip('()'), errors='coerce')) or x in ['-','–']))
+    return sum(1 for x in row if (pd.notna(pd.to_numeric(str(x).replace(",", "").strip('()'), errors='coerce')) or x in ['-','–']))
 
 
 def convert_to_numeric(value):
@@ -57,23 +67,32 @@ def convert_to_numeric(value):
         value = '-' + value[1:-1]
 
     if all(char.isdigit() or char in '-,.' for char in str(value)):
-        cleaned_value = pd.to_numeric(value.replace(',', ''), errors='coerce')
+        cleaned_value = pd.to_numeric(str(value).replace(',', ''), errors='coerce')
         return cleaned_value
     return value
 
 def get_headers(dataframes):
     # Get the dataframe columns name
-    if len(dataframes) >= 2:
-        df_for_columns_names = dataframes[1]
-    else:
-        df_for_columns_names = dataframes[0]
-    for i, row in df_for_columns_names.iterrows():
-        numeric_count = get_numeric_count(row)
-        if numeric_count >= 2:
-            first_numeric_idx = i
-            break
+    # if len(dataframes) >= 2:
+    #     df_for_columns_names = dataframes[1]
+    # else:
+    #     df_for_columns_names = dataframes[0]
+    first_numeric_idx = None
+
+    order = list(range(1, len(dataframes))) + [0]
+
+    for k in order:
+        if first_numeric_idx is None:
+            df_for_columns_names = dataframes[k]
+            df_for_columns_names = df_for_columns_names.astype(str).where(pd.notna(df_for_columns_names), "")
+            for i, row in df_for_columns_names.iterrows():
+                numeric_count = get_numeric_count(row)
+                if numeric_count >= 2:
+                    first_numeric_idx = i
+                    break
+        if first_numeric_idx is not None:
+            break 
 
-    df_for_columns_names = df_for_columns_names.astype(str).where(pd.notna(df_for_columns_names), "")
 
     new_header = [" ".join(filter(None, df_for_columns_names.iloc[:first_numeric_idx, col].values)) for col in range(df_for_columns_names.shape[1])]
 
@@ -81,7 +100,12 @@ def get_headers(dataframes):
 
 def clean_dataframe(df,header):
     # Rule : if a row is not numerical, merge it with the next numerical one
-    df.columns = header
+    if len(header) < len(df.columns):
+        df.columns = header + [f"Unnamed_{i}" for i in range(len(header), len(df.columns))]
+    elif len(header) > len(df.columns):
+        df.columns = header[:len(df.columns)]
+    else:
+        df.columns = header
     first_numeric_idx = None
     for i, row in df.iterrows():
         numeric_count = get_numeric_count(row)
@@ -104,14 +128,14 @@ def clean_dataframe(df,header):
                 buffer = list(df.iloc[i].copy())
             else:
                 buffer = [
-                    " ".join(filter(lambda x: x not in [None, "None", ""], [buffer[j], df.iloc[i, j]]))
+                    " ".join(filter(lambda x: x not in [None, "None", ""], [str(buffer[j]), str(df.iloc[i, j])]))
                     for j in range(df.shape[1])
                 ]
             merged_rows.append(i)
         else:
             if buffer is not None:
                 df.iloc[i] = [
-                    " ".join(filter(lambda x: x not in [None, "None", ""], [buffer[j], df.iloc[i, j]]))
+                    " ".join(filter(lambda x: x not in [None, "None", ""], [str(buffer[j]), str(df.iloc[i, j])]))
                     for j in range(df.shape[1])
                 ]
                 buffer = None
@@ -120,9 +144,7 @@ def clean_dataframe(df,header):
     return clean_df
 
 
-def clean_and_concatenate_tables(tables):
-    dataframes = [table.df for table in tables]
-
+def clean_and_concatenate_tables(dataframes):
     for i in range(len(dataframes)):
         df = dataframes[i]
         row_counts = df.apply(lambda row: row.notna().sum() - (row.astype(str) == "").sum(), axis=1)
@@ -130,28 +152,57 @@ def clean_and_concatenate_tables(tables):
         dataframes[i] = df.loc[row_counts >= 1, col_counts >= 3].reset_index(drop = True)
 
     new_header = get_headers(dataframes) 
-
     cleaned_dfs = []
 
     for df in dataframes:
-        cleaned_dfs.append(clean_dataframe(df,new_header))
+        if len(df.columns) >= 3 : 
+            cleaned_dfs.append(clean_dataframe(df,new_header))
+    
+    cleaned_dfs = [df.reset_index(drop=True) for df in cleaned_dfs if isinstance(df, pd.DataFrame) and not df.empty]
+
+    if not cleaned_dfs:
+        raise ValueError("After cleaning, no valid dataframe left.")
+
+    for _, df in enumerate(cleaned_dfs):
+        if any(col == '' for col in df.columns):  # Check if there are empty column names
+            df.columns = [f"col_{j}" if col == '' else col for j, col in enumerate(df.columns)]
 
     concatenated_df = pd.concat(cleaned_dfs, ignore_index=True)
+
+    if concatenated_df.shape[0] <= 4 :
+        raise ValueError("Dataframe too small, probable mistake")
+    if concatenated_df.shape[1] <= 2 :
+        raise ValueError("Less than 3 columns, probable mistake")
+    print("Success of conversion : dataframe of shape ",concatenated_df.shape)
     return concatenated_df
 
 
+
 def convert_to_excel(reduced_pdf_folder, output_folder):
     if os.path.exists(output_folder):
         shutil.rmtree(output_folder)
     os.makedirs(output_folder)
     
+    failed_folder = os.path.join(output_folder, "failed_to_convert")
+    if os.path.exists(failed_folder):
+        shutil.rmtree(failed_folder)
+    os.makedirs(failed_folder)
+    
+    if os.path.exists("./log_errors.txt"):
+        os.remove("./log_errors.txt")
+
+    number_of_files = 0
+    number_of_fails = 0
     for filename in os.listdir(reduced_pdf_folder):
         if filename.endswith('.pdf'):
+            number_of_files += 1
+            print("Trying to convert :", filename, "to excel")
             pdf_path = os.path.join(reduced_pdf_folder, filename)
-            tables = extract_tables_camelot(pdf_path)
-            if tables:
-                concatenated_df = clean_and_concatenate_tables(tables)
-
+            try:
+                dataframes = extract_tables_camelot_or_tabula(pdf_path)
+                if not dataframes:
+                    raise ValueError(f'No tables found in {filename}')
+                concatenated_df = clean_and_concatenate_tables(dataframes)
                 excel_path = os.path.join(output_folder, filename.replace('.pdf', '.xlsx'))
 
                 for col in concatenated_df.columns:
@@ -172,7 +223,7 @@ def convert_to_excel(reduced_pdf_folder, output_folder):
                             ws.cell(row=r_idx + 1, column=c_idx + 1, value=numeric_value)
                             percentage_cells.append((r_idx + 1, c_idx + 1)) 
 
-                tab = Table(displayName = "Table1",ref=ws.dimensions)
+                tab = Table(displayName="Table1", ref=ws.dimensions)
                 style = TableStyleInfo(
                     name="TableStyleMedium9",
                     showFirstColumn=False,
@@ -186,22 +237,29 @@ def convert_to_excel(reduced_pdf_folder, output_folder):
 
                 # Ajuster la largeur des colonnes
                 for column_cells in ws.columns:
-                    length = min(max(len(str(cell.value)) for cell in column_cells),30)
+                    length = min(max(len(str(cell.value)) for cell in column_cells), 30)
                     ws.column_dimensions[column_cells[0].column_letter].width = length + 2
 
                 for row, col in percentage_cells:
                     cell = ws.cell(row=row, column=col)
                     cell.number_format = numbers.BUILTIN_FORMATS[10]
                 wb.save(excel_path)          
-                print(f'Saved {filename} as Excel file')
-            else:
-                print(f'No tables found in {filename}')
-    shutil.make_archive(base_name="./output", format='zip', root_dir="./outputs")
+            except Exception as e:
+                error_message = f"Error converting {filename}: {e}"
+                print(error_message)
+                number_of_fails += 1
+                shutil.copy(pdf_path, os.path.join(failed_folder, filename))
+                with open("./log_errors.txt", "a") as log_file:
+                    log_file.write(error_message + "\n")
+    print("Number of files considered : ",number_of_files)
+    print("Number of success : ", number_of_files - number_of_fails)
+    print("Number of fails : ", number_of_fails)
+    shutil.make_archive(base_name="./output", format='zip', root_dir=output_folder)
 
 
 def reduce_and_convert(input_folder):
     reduced_pdf_folder = "./reduced_pdf"
-    output_folder = './outputs'
+    output_folder = "./outputs"
     reduce_pdf(input_folder,reduced_pdf_folder)
     convert_to_excel(reduced_pdf_folder, output_folder)
 

requirements.txt

@@ -2,3 +2,5 @@ PyPDF2
 pandas
 camelot-py
 openpyxl
+PyCryptodome
+tabula