Update app.py

app.py

@@ -58,23 +58,22 @@ class PaperDownloader:
                 logger.debug(f"Error fetching {url}: {e}")
                 return None, None
 
-
     async def fetch_pdf_content(self, session, url, max_redirects=5, max_retries=2, retry_delay=1):
-            """Fetch content and validate if response is PDF, following up to max_redirects redirections with retries."""
-            
-            current_url = url
-            redirect_count = 0
-            retry_count = 0
+        """Fetch content and validate if response is PDF, following up to max_redirects redirections with retries."""
 
-            while redirect_count <= max_redirects:
-                try:
-                  while retry_count <= max_retries:
+        current_url = url
+        redirect_count = 0
+        retry_count = 0
+
+        while redirect_count <= max_redirects:
+            try:
+                while retry_count <= max_retries:
                     try:
                         async with session.get(current_url, headers=self.headers, timeout=10, allow_redirects=False) as response:
 
-                            if response.status in [301,302, 307,308]:
+                            if response.status in [301, 302, 307, 308]:
                                 current_url = response.headers['Location']
-                                redirect_count+=1
+                                redirect_count += 1
                                 logger.debug(f"Following redirect from {url} to {current_url}")
                                 break  # Break out of the retry loop for a redirect
 
@@ -83,61 +82,63 @@ class PaperDownloader:
                             if 'application/pdf' in response.headers.get('Content-Type', ''):
                                 return await response.read()
                             else:
-                              logger.debug(f"Content type not PDF for {current_url}: {response.headers.get('Content-Type', '')}")
-                              return None
+                                logger.debug(
+                                    f"Content type not PDF for {current_url}: {response.headers.get('Content-Type', '')}")
+                                return None
                     except Exception as e:
-                         logger.debug(f"Error getting PDF, retrying ({retry_count}/{max_retries}) from {current_url}: {e}")
-                         retry_count+=1
-                         await asyncio.sleep(retry_delay)
+                        logger.debug(
+                            f"Error getting PDF, retrying ({retry_count}/{max_retries}) from {current_url}: {e}")
+                        retry_count += 1
+                        await asyncio.sleep(retry_delay)
 
-                  retry_count = 0 # Reset the retry count, in case there's a next redirect attempt
-                  
-                except Exception as e:
-                   logger.debug(f"Error getting PDF from {current_url}: {e}")
-                   return None
-                
-            logger.debug(f"Too many redirects or retries {url}, not following this link further")
-            return None
+                retry_count = 0  # Reset the retry count, in case there's a next redirect attempt
+
+            except Exception as e:
+                logger.debug(f"Error getting PDF from {current_url}: {e}")
+                return None
+
+        logger.debug(f"Too many redirects or retries {url}, not following this link further")
+        return None
         
     async def download_paper_direct_doi_async(self, session, doi):
-            """Attempt to download the pdf from the landing page of the doi"""
-            if not doi:
-               return None
+        """Attempt to download the pdf from the landing page of the doi"""
+        if not doi:
+           return None
             
-            try:
-                 doi_url = f"https://doi.org/{self.clean_doi(doi)}"
-
-                 # First, let's try to download the URL directly in case it is already the pdf.
-                 pdf_content = await self.fetch_pdf_content(session, doi_url)
-                 if pdf_content:
-                   logger.debug(f"Direct DOI resolved to PDF from {doi_url}")
-                   return pdf_content
-
-                 # If direct DOI link was not a pdf, fetch landing page and extract links
-                 text, headers = await self.fetch_with_headers(session, doi_url, timeout=15)
-                 if not text:
-                    return None
-
-                 pdf_patterns = [
-                    r'(https?://[^\s<>"]+?\.pdf)',
-                    r'(https?://[^\s<>"]+?download/[^\s<>"]+)',
-                    r'(https?://[^\s<>"]+?\/pdf\/[^\s<>"]+)',
-                    ]
-                 
-                 pdf_urls = []
-                 for pattern in pdf_patterns:
-                    pdf_urls.extend(re.findall(pattern, text))
-
-                 # Attempt each pdf url and break when you find a PDF content.
-                 for pdf_url in pdf_urls:
-                    pdf_content = await self.fetch_pdf_content(session, pdf_url)
-                    if pdf_content:
-                      logger.debug(f"Found PDF from: {pdf_url}")
-                      return pdf_content
+        try:
+             doi_url = f"https://doi.org/{self.clean_doi(doi)}"
     
-            except Exception as e:
-                logger.debug(f"Error trying to get the PDF from {doi}: {e}")
+             # First, let's try to download the URL directly in case it is already the pdf.
+             pdf_content = await self.fetch_pdf_content(session, doi_url)
+             if pdf_content:
+                logger.debug(f"Direct DOI resolved to PDF from {doi_url}")
+                return pdf_content
+    
+             # If direct DOI link was not a pdf, fetch landing page and extract links
+             text, headers = await self.fetch_with_headers(session, doi_url, timeout=15)
+             if not text:
                 return None
+    
+             pdf_patterns = [
+                r'(https?://[^\s<>"]+?\.pdf)',
+                r'(https?://[^\s<>"]+?download/[^\s<>"]+)',
+                r'(https?://[^\s<>"]+?\/pdf\/[^\s<>"]+)',
+                ]
+                 
+             pdf_urls = []
+             for pattern in pdf_patterns:
+                pdf_urls.extend(re.findall(pattern, text))
+    
+             # Attempt each pdf url and break when you find a PDF content.
+             for pdf_url in pdf_urls:
+                pdf_content = await self.fetch_pdf_content(session, pdf_url)
+                if pdf_content:
+                  logger.debug(f"Found PDF from: {pdf_url}")
+                  return pdf_content
+    
+        except Exception as e:
+            logger.debug(f"Error trying to get the PDF from {doi}: {e}")
+            return None
         
     async def download_paper_scihub_async(self, session, doi):
             """Improved method to download paper from Sci-Hub using async requests"""
@@ -169,7 +170,7 @@ class PaperDownloader:
                          if pdf_content:
                             logger.debug(f"Found PDF from: {pdf_url}")
                             return pdf_content
-
+    
                 except Exception as e:
                     logger.debug(f"Error trying to download {doi} from {base_url}: {e}")
     
@@ -179,20 +180,20 @@ class PaperDownloader:
         """Download from Libgen, handles the query and the redirection"""
         if not doi:
             return None
-
+    
         base_url = 'https://libgen.rs/scimag/'
         try:
             search_url = f"{base_url}?q={self.clean_doi(doi)}"
             text, headers = await self.fetch_with_headers(session, search_url, timeout=10)
-
+    
             if not text or "No results" in text:
                 logger.debug(f"No results for DOI: {doi} on libgen")
                 return None
-
+    
             soup = BeautifulSoup(text, 'html.parser')
-
+    
             links = soup.select('table.c > tbody > tr:nth-child(2) > td:nth-child(1) > a')
-
+    
             if links:
                 link = links[0]
                 pdf_url = link['href']
@@ -205,36 +206,36 @@ class PaperDownloader:
         return None
 
     async def download_paper_google_scholar_async(self, session, doi):
-        """Search google scholar to find an article with the given doi, try to get the pdf"""
-        if not doi:
-            return None
-
-        try:
-            query = f'doi:"{doi}"'
-            params = {'q': query}
-            url = f'https://scholar.google.com/scholar?{urlencode(params)}'
-
-            text, headers = await self.fetch_with_headers(session, url, timeout=10)
-            if not text:
+            """Search google scholar to find an article with the given doi, try to get the pdf"""
+            if not doi:
                 return None
-
-            soup = BeautifulSoup(text, 'html.parser')
-
-            # Find any links with [PDF]
-            links = soup.find_all('a', string=re.compile(r'\[PDF\]', re.IGNORECASE))
-
-            if links:
-                pdf_url = links[0]['href']
-                pdf_content = await self.fetch_pdf_content(session,pdf_url)
-                if pdf_content:
-                  logger.debug(f"Found PDF from: {pdf_url}")
-                  return pdf_content
-
-        except Exception as e:
-            logger.debug(f"Google Scholar error for {doi}: {e}")
-
-        return None
     
+            try:
+                query = f'doi:"{doi}"'
+                params = {'q': query}
+                url = f'https://scholar.google.com/scholar?{urlencode(params)}'
+    
+                text, headers = await self.fetch_with_headers(session, url, timeout=10)
+                if not text:
+                    return None
+    
+                soup = BeautifulSoup(text, 'html.parser')
+    
+                # Find any links with [PDF]
+                links = soup.find_all('a', string=re.compile(r'\[PDF\]', re.IGNORECASE))
+    
+                if links:
+                    pdf_url = links[0]['href']
+                    pdf_content = await self.fetch_pdf_content(session,pdf_url)
+                    if pdf_content:
+                      logger.debug(f"Found PDF from: {pdf_url}")
+                      return pdf_content
+    
+            except Exception as e:
+                logger.debug(f"Google Scholar error for {doi}: {e}")
+    
+            return None
+
     async def download_paper_crossref_async(self, session, doi):
             """Alternative search method using Crossref"""
             if not doi:
@@ -262,7 +263,7 @@ class PaperDownloader:
             except Exception as e:
                 logger.debug(f"Crossref error for {doi}: {e}")
             return None
-    
+
     async def download_with_retry_async(self, doi, max_retries=3, initial_delay=2):
         """Downloads a paper using multiple strategies with exponential backoff and async requests"""
         pdf_content = None
@@ -292,7 +293,7 @@ class PaperDownloader:
                     delay *= 2  # Exponential backoff
 
         return None
-    
+        
     async def download_single_doi_async(self, doi):
             """Downloads a single paper using a DOI"""
             if not doi:
@@ -303,20 +304,21 @@ class PaperDownloader:
     
                 if pdf_content:
                     if doi is None:
-                        return None, "Error: DOI not provided", "Error: DOI not provided"
+                         return None, "Error: DOI not provided", "Error: DOI not provided"
                     filename = f"{str(doi).replace('/', '_').replace('.', '_')}.pdf"
                     filepath = os.path.join(self.output_dir, filename)
-                    with open(filepath, 'wb') as f:
-                        f.write(pdf_content)
+                    #write the file asynchronously here so it doesn't block
+                    loop = asyncio.get_running_loop()
+                    await loop.run_in_executor(None, lambda:  open(filepath, 'wb').write(pdf_content))
                     logger.info(f"Successfully downloaded: {filename}")
                     return filepath, f'<a href="https://doi.org/{doi}">{doi}</a>', ""
                 else:
-                    logger.warning(f"Could not download: {doi}")
-                    return None, f"Could not download {doi}", f'<a href="https://doi.org/{doi}">{doi}</a>'
+                     logger.warning(f"Could not download: {doi}")
+                     return None, f"Could not download {doi}", f'<a href="https://doi.org/{doi}">{doi}</a>'
     
             except Exception as e:
-                logger.error(f"Error processing {doi}: {e}")
-                return None, f"Error processing {doi}: {e}", f"Error processing {doi}: {e}"
+                 logger.error(f"Error processing {doi}: {e}")
+                 return None, f"Error processing {doi}: {e}", f"Error processing {doi}: {e}"
     
     async def download_multiple_dois_async(self, dois_text):
             """Downloads multiple papers from a list of DOIs"""
@@ -344,13 +346,18 @@ class PaperDownloader:
     
             if downloaded_files:
                 zip_filename = 'papers.zip'
-                with zipfile.ZipFile(zip_filename, 'w') as zipf:
-                    for file_path in downloaded_files:
-                        zipf.write(file_path, arcname=os.path.basename(file_path))
+                # Zip asynchronously
+                loop = asyncio.get_running_loop()
+                await loop.run_in_executor(None, lambda:  self.create_zip(zip_filename,downloaded_files) )
                 logger.info(f"ZIP file created: {zip_filename}")
     
             return zip_filename if downloaded_files else None, "\n".join(downloaded_links), "\n".join(failed_dois)
-    
+
+    def create_zip(self, zip_filename, downloaded_files):
+      with zipfile.ZipFile(zip_filename, 'w') as zipf:
+            for file_path in downloaded_files:
+                zipf.write(file_path, arcname=os.path.basename(file_path))
+
     async def process_bibtex_async(self, bib_file):
             """Process BibTeX file and download papers with multiple strategies"""
             # Read BibTeX file content from the uploaded object
@@ -389,10 +396,11 @@ class PaperDownloader:
                             return None, "Error: DOI not provided", "Error: DOI not provided"
                         filename = f"{str(doi).replace('/', '_').replace('.', '_')}_{i}.pdf"
                         filepath = os.path.join(self.output_dir, filename)
-    
-                        with open(filepath, 'wb') as f:
-                            f.write(pdf_content)
-    
+
+                        #Write the file asynchronously so it doesn't block the ui.
+                        loop = asyncio.get_running_loop()
+                        await loop.run_in_executor(None, lambda:  open(filepath, 'wb').write(pdf_content))
+                        
                         downloaded_files.append(filepath)
                         downloaded_links.append(f'<a href="https://doi.org/{doi}">{doi}</a>')
                         logger.info(f"Successfully downloaded: {filename}")
@@ -406,13 +414,13 @@ class PaperDownloader:
             # Create ZIP of downloaded papers
             if downloaded_files:
                 zip_filename = 'papers.zip'
-                with zipfile.ZipFile(zip_filename, 'w') as zipf:
-                    for file_path in downloaded_files:
-                        zipf.write(file_path, arcname=os.path.basename(file_path))
+                # Zip asynchronously so the main loop is not blocked.
+                loop = asyncio.get_running_loop()
+                await loop.run_in_executor(None, lambda:  self.create_zip(zip_filename,downloaded_files) )
                 logger.info(f"ZIP file created: {zip_filename}")
     
             return zip_filename, "\n".join(downloaded_links), "\n".join(failed_dois)
-    
+
 
 def create_gradio_interface():
     """Create Gradio interface for Paper Downloader"""
@@ -434,6 +442,7 @@ def create_gradio_interface():
             return zip_path, downloaded_dois, failed_dois, None
         else:
             return None, "Please provide a .bib file, a single DOI, or a list of DOIs", "Please provide a .bib file, a single DOI, or a list of DOIs", None
+
     # Gradio Interface
     interface = gr.Interface(
         fn=download_papers,