Update app.py

app.py

@@ -27,6 +27,7 @@ HF_TOKEN = os.getenv('HF_TOKEN')
 #==========================================================================================================
 
 def format_table_data(data_string):
+    
     # Split the string into rows based on newlines
     rows = data_string.strip().split("\n")
 
@@ -37,21 +38,29 @@ def format_table_data(data_string):
         # Split each row into columns based on the separator '|' and strip extra spaces
         columns = row.split("|")
         formatted_row = [cell.strip() for cell in columns]
-        
-        # Handle uneven rows by ensuring each row has the same number of columns
-        max_columns = max(len(columns) for columns in formatted_data) if formatted_data else len(columns)
-        while len(formatted_row) < max_columns:
-            formatted_row.append("")  # Add empty strings to fill the row
-        
+
         formatted_data.append(formatted_row)
     
-    # Handle case where new rows have more columns than previous rows
-    max_columns = max(len(row) for row in formatted_data)
-    for row in formatted_data:
+    # Determine the minimum and maximum number of elements that a column must have
+    min_elements = len(formatted_data) * 0.5  # For example, at least half the rows
+    max_elements = len(formatted_data) * 1.5  # For example, no more than 1.5 times the number of rows
+
+    # Transpose the data to work with columns
+    transposed_data = list(map(list, zip(*formatted_data)))
+
+    # Filter out outlier columns
+    filtered_columns = [col for col in transposed_data if min_elements <= len(col) <= max_elements]
+
+    # Transpose the data back to the original format
+    filtered_data = list(map(list, zip(*filtered_columns)))
+
+    # Handle uneven rows by ensuring each row has the same number of columns
+    max_columns = max(len(row) for row in filtered_data)
+    for row in filtered_data:
         while len(row) < max_columns:
             row.append("")  # Add empty strings to fill the row
 
-    return formatted_data
+    return filtered_data
 
 #==========================================================================================================