more pdf text parsing

app.py

@@ -23,13 +23,24 @@ client_anthropic = Anthropic(api_key=anthropic_api_key)
 
 def parse_quotes(input_string, pdf_text):
 
-    pdf_text_wo_header = re.sub(r'(?m)^Under review as a conference paper at ICLR 2024*$', '', pdf_text)
+    header_pattern = r'\d+Under review as a conference paper at ICLR 2024'
+    #r'^\d+Under review as a conference paper at ICLR 2024\s*'  # Pattern to match headers with variable page numbers
+    # r'(?m)^Under review as a conference paper at ICLR 2024*$'
+
+    pdf_text_wo_header = re.sub(header_pattern, '', pdf_text)
+    # Remove new lines that are not followed by a period, exclamation mark, or question mark
+    pdf_text_wo_header = re.sub(r'(?<!\.\s)(?<!\!\s)(?<!\?\s)\n+', ' ', pdf_text_wo_header)
+    # Remove extra spaces that may have been introduced
+    pdf_text_wo_header = re.sub(r'\s{2,}', ' ', pdf_text_wo_header).strip()
     
     # Find all matches of <quote>...</quote> and extract the content between the tags
     matches = re.findall(r'<quote>(.*?)</quote>', input_string)
     
     count = len(matches)
     extracted_texts = matches
+    # with open('pdf_text.txt', 'w') as file:
+    #     file.write(pdf_text_wo_header)
+    # print(extracted_texts)
     match_count = sum(1 for text in extracted_texts if text in pdf_text_wo_header)
     
     return count, match_count