Update app.py

app.py

@@ -161,7 +161,6 @@ class PaperDownloader:
                 text, headers = await self.fetch_with_headers(session, scihub_url, timeout=15)
                 if not text:
                     continue
-                    
 
                 # Search for multiple PDF URL patterns
                 pdf_patterns = [
@@ -169,24 +168,25 @@ class PaperDownloader:
                     r'(https?://[^\s<>"]+?download/[^\s<>"]+)',
                     r'(https?://[^\s<>"]+?\/pdf\/[^\s<>"]+)',
                 ]
-            
 
                 pdf_urls = []
                 for pattern in pdf_patterns:
                     pdf_urls.extend(re.findall(pattern, text))
-            
 
-                # Try downloading from found URLs, but iterate over ALL
+                # Try downloading from found URLs
                 for pdf_url in pdf_urls:
-                    pdf_content = await self.fetch_pdf_content(session,pdf_url)
-                    if pdf_content:
-                        logger.debug(f"Found PDF from: {pdf_url}")
-                        return pdf_content
-                    
+                    try:
+                        pdf_response = await session.get(pdf_url, headers=self.headers, timeout=10)
+                        # Verify if it's a PDF
+                        if 'application/pdf' in pdf_response.headers.get('Content-Type', ''):
+                            logger.debug(f"Found PDF from: {pdf_url}")
+                            return await pdf_response.read()
+                    except Exception as e:
+                        logger.debug(f"Error downloading PDF from {pdf_url}: {e}")
+
             except Exception as e:
-                 logger.debug(f"Error trying to download {doi} from {base_url}: {e}")
+                logger.debug(f"Error trying to download {doi} from {base_url}: {e}")
 
-        
         return None
 
     async def download_paper_libgen_async(self, session, doi):
@@ -210,10 +210,10 @@ class PaperDownloader:
             if links:
                 link = links[0]
                 pdf_url = link['href']
-                pdf_content = await self.fetch_pdf_content(session, pdf_url)
-                if pdf_content:
+                pdf_response = await session.get(pdf_url, headers=self.headers, allow_redirects=True, timeout=10)
+                if 'application/pdf' in pdf_response.headers.get('Content-Type', ''):
                     logger.debug(f"Found PDF from: {pdf_url}")
-                    return pdf_content
+                    return await pdf_response.read()
         except Exception as e:
             logger.debug(f"Error trying to download {doi} from libgen: {e}")
         return None
@@ -239,99 +239,77 @@ class PaperDownloader:
 
             if links:
                 pdf_url = links[0]['href']
-                pdf_content = await self.fetch_pdf_content(session,pdf_url)
-                if pdf_content:
-                  logger.debug(f"Found PDF from: {pdf_url}")
-                  return pdf_content
-
+                pdf_response = await session.get(pdf_url, headers=self.headers, timeout=10)
+                if 'application/pdf' in pdf_response.headers.get('Content-Type', ''):
+                    logger.debug(f"Found PDF from: {pdf_url}")
+                    return await pdf_response.read()
         except Exception as e:
             logger.debug(f"Google Scholar error for {doi}: {e}")
 
         return None
-    
-    async def download_paper_crossref_async(self, session, doi):
-            """Alternative search method using Crossref"""
-            if not doi:
-                return None
-    
-            try:
-                # Search for open access link
-                url = f"https://api.crossref.org/works/{doi}"
-                response = await session.get(url, headers=self.headers, timeout=10)
-    
-                if response.status == 200:
-                    data = await response.json()
-                    work = data.get('message', {})
-    
-                    # Search for open access links
-                    links = work.get('link', [])
-                    for link in links:
-                        if link.get('content-type') == 'application/pdf':
-                            pdf_url = link.get('URL')
-                            if pdf_url:
-                                pdf_content = await self.fetch_pdf_content(session, pdf_url)
-                                if pdf_content:
-                                    logger.debug(f"Found PDF from: {pdf_url}")
-                                    return pdf_content
-            except Exception as e:
-                logger.debug(f"Crossref error for {doi}: {e}")
-            return None
 
-    async def download_with_retry_async(self, doi, max_retries=3):
-        """
-        Intenta descargar un paper con múltiples estrategias y un número limitado de reintentos.
-        
-        Args:
-            doi (str): DOI del paper a descargar
-            max_retries (int): Número máximo de reintentos
-        
-        Returns:
-            bytes or None: Contenido del PDF o None si no se puede descargar
-        """
+    async def download_paper_crossref_async(self, session, doi):
+        """Alternative search method using Crossref"""
         if not doi:
-            logger.warning("DOI no proporcionado")
             return None
-    
-        # Estrategias de descarga en orden de preferencia
-        download_strategies = [
-            self.download_paper_direct_doi_async,
-            self.download_paper_scihub_async,
-            self.download_paper_libgen_async,
-            self.download_paper_google_scholar_async,
-            self.download_paper_crossref_async
-        ]
-    
-        async with aiohttp.ClientSession() as session:
-            for retry in range(max_retries):
-                logger.info(f"Intento de descarga {retry + 1} para DOI: {doi}")
-                
-                # Probar cada estrategia de descarga
-                for strategy in download_strategies:
-                    try:
-                        logger.info(f"Trying strategy {strategy.__name__} for DOI {doi}") # ADDED
-                        pdf_content = await strategy(session, doi)
-                        if pdf_content:
-                            logger.info(f"Descarga exitosa de {doi} usando {strategy.__name__}")
-                            return pdf_content
-                    except CancelledError:
-                         logger.info(f"Download cancelled on strategy: {strategy.__name__} with DOI {doi}")
-                         return None # return here in order to stop retry
 
-                    except Exception as e:
-                        logger.debug(f"Error en estrategia {strategy.__name__} para {doi}: {e}") #ADDED
-                    
-                    
-                # Si ninguna estrategia funcionó, esperar un poco antes de reintentar
-                await asyncio.sleep(1)  # Pequeña pausa entre reintentos
-                
-                if retry == max_retries -1: #log all if it does not works on max retries.
-                      logger.warning(f"FALLO FINAL: No se pudo descargar DOI {doi} después de {max_retries} intentos")
+        try:
+            # Search for open access link
+            url = f"https://api.crossref.org/works/{doi}"
+            response = await session.get(url, headers=self.headers, timeout=10)
+
+            if response.status == 200:
+                data = await response.json()
+                work = data.get('message', {})
+
+                # Search for open access links
+                links = work.get('link', [])
+                for link in links:
+                    if link.get('content-type') == 'application/pdf':
+                        pdf_url = link.get('URL')
+                        if pdf_url:
+                            pdf_response = await session.get(pdf_url, headers=self.headers)
+                            if 'application/pdf' in pdf_response.headers.get('Content-Type', ''):
+                                logger.debug(f"Found PDF from: {pdf_url}")
+                                return await pdf_response.read()
 
+        except Exception as e:
+            logger.debug(f"Crossref error for {doi}: {e}")
 
-            # Si se agotan todos los reintentos
-            return None
+        return None
+
+    async def download_with_retry_async(self, doi, max_retries=3, initial_delay=2):
+        """Downloads a paper using multiple strategies with exponential backoff and async requests"""
+        pdf_content = None
+        retries = 0
+        delay = initial_delay
 
-    async def _download_single_doi(self, doi, cancel_event):
+        async with aiohttp.ClientSession() as session:
+            while retries < max_retries and not pdf_content:
+                try:
+                    pdf_content = (
+                        await self.download_paper_direct_doi_async(session, doi) or
+                        await self.download_paper_scihub_async(session, doi) or
+                        await self.download_paper_libgen_async(session, doi) or
+                        await self.download_paper_google_scholar_async(session, doi) or
+                        await self.download_paper_crossref_async(session, doi)
+
+                    )
+                    if pdf_content:
+                        return pdf_content
+                except Exception as e:
+                    logger.error(f"Error in download attempt {retries + 1} for DOI {doi}: {e}")
+
+                if not pdf_content:
+                    retries += 1
+                    logger.warning(f"Retry attempt {retries} for DOI: {doi} after {delay} seconds")
+                    await asyncio.sleep(delay)
+                    delay *= 2  # Exponential backoff
+
+        return None
+
+    
+    async def _download_single_doi(self, doi):
             """Descargar un único DOI con retroalimentación de progreso"""
             if not doi:
                 return None, "Error: DOI no proporcionado", "Error: DOI no proporcionado"
@@ -348,7 +326,7 @@ class PaperDownloader:
                     # Escribir contenido del PDF
                     
                     with open(filepath, 'wb') as f:
-                       f.write(pdf_content)
+                        f.write(pdf_content)
                        
                     logger.info(f"Saved PDF to file: {filepath}")
 
@@ -357,13 +335,12 @@ class PaperDownloader:
                     return filepath, f"Descargado exitosamente: <a href='https://doi.org/{doi}'>{doi}</a>", ""
                    
                 else:
-                    logger.warning(f"No se pudo descargar: {doi}")
-                    return None, f"No se pudo descargar {doi}", f'<a href="https://doi.org/{doi}">{doi}</a>'
+                     logger.warning(f"No se pudo descargar: {doi}")
+                     return None, f"No se pudo descargar {doi}", f'<a href="https://doi.org/{doi}">{doi}</a>'
            
             except CancelledError:
                 logger.info(f"Download Cancelled DOI: {doi}")
                 return None, f"Download cancelled {doi}","Download Cancelled"
-            
             except Exception as e:
                 logger.error(f"Error processing {doi}: {e}")
                 return None, f"Error processing {doi}: {e}", f"Error processing {doi}: {e}"
@@ -380,7 +357,7 @@ class PaperDownloader:
             
             # Validar lista de DOIs
             if not dois:
-               return None, "Error: No valid DOIs provided", "Error: No valid DOIs provided", ""
+                return None, "Error: No valid DOIs provided", "Error: No valid DOIs provided", ""
     
             # Listas para rastrear resultados
             downloaded_files = []      # Rutas de archivos descargados
@@ -389,44 +366,47 @@ class PaperDownloader:
             
             for i, doi in enumerate(dois):
                     result = await self._download_single_doi(doi,cancel_event)
+
                     if cancel_event.is_set():
                         logger.info("Downloads cancelled on multiple dois download")
-                        return  None,"Downloads cancelled","Downloads cancelled", "" # early return on cancelled
+                        return None, "Downloads cancelled","Downloads cancelled", "" # early return on cancelled
+                    
                     if result is None:
-                        continue
+                       continue;
 
                     if isinstance(result, Exception):
                         # Excepción inesperada
-                        error_msg = f"Unexpected error: {str(result)}"
-                        logger.error(f"Error downloading {doi}: {error_msg}")
-                        failed_dois.append(f'<a href="https://doi.org/{doi}">{doi}</a> - {error_msg}')
-                    
+                         error_msg = f"Unexpected error: {str(result)}"
+                         logger.error(f"Error downloading {doi}: {error_msg}")
+                         failed_dois.append(f'<a href="https://doi.org/{doi}">{doi}</a> - {error_msg}')
+                   
                     elif result[0] is None:
                         # Descarga fallida (resultado de download_single_doi_async)
                         error_msg = result[1]
                         logger.warning(f"Failed to download {doi}: {error_msg}")
                         failed_dois.append(f'<a href="https://doi.org/{doi}">{doi}</a> - {error_msg}')
-                    
+                   
                     else:
                          # Descarga exitosa
                         filepath = result[0]
-                        
+                         
                          # Generar nombre de archivo único
-                        filename = f"{str(doi).replace('/', '_').replace('.', '_')}.pdf" #Fixed indent
-                        filepath_unique = os.path.join(self.output_dir, filename) # Fixed indent
+                        filename = f"{str(doi).replace('/', '_').replace('.', '_')}.pdf"  # indent problem corrected
+                        filepath_unique = os.path.join(self.output_dir, filename) #Fixed identation.
                          
                         try:
-                             # Renombrar archivo
-                           os.rename(filepath, filepath_unique) #Fixed indent
+                              # Renombrar archivo
+                             os.rename(filepath, filepath_unique)  #Fixed identation.
                             
                             # Añadir a lista de archivos descargados
-                           downloaded_files.append(filepath_unique)  #Fixed indent
-                           downloaded_links.append(f'<a href="https://doi.org/{doi}">{doi}</a>') #Fixed indent
+                             downloaded_files.append(filepath_unique) #Fixed identation.
+                             downloaded_links.append(f'<a href="https://doi.org/{doi}">{doi}</a>')#Fixed identation.
                         
                          except Exception as rename_error:
-                            logger.error(f"Error renaming file for {doi}: {rename_error}")
-                            failed_dois.append(f'<a href="https://doi.org/{doi}">{doi}</a> - Error saving file') #Fixed indent
-            
+                               logger.error(f"Error renaming file for {doi}: {rename_error}")
+                               failed_dois.append(f'<a href="https://doi.org/{doi}">{doi}</a> - Error saving file')#Fixed identation.
+
+                    
             # Crear archivo ZIP si hay archivos descargados
             zip_filename = None
             if downloaded_files:
@@ -439,8 +419,9 @@ class PaperDownloader:
                     lambda: self.create_zip(zip_filename, downloaded_files)
                 )
                 logger.info(f"ZIP file created: {zip_filename}")
-            return zip_filename if downloaded_files else None, "\n".join(downloaded_links),"\n".join(failed_dois),""
 
+            return  zip_filename if downloaded_files else None, "\n".join(downloaded_links),"\n".join(failed_dois), ""
+    
     async def process_bibtex(self, bib_file, cancel_event):
             """Process BibTeX file and download papers with multiple strategies and reports UI updates using a callback"""
             # Read BibTeX file content from the uploaded object
@@ -449,7 +430,7 @@ class PaperDownloader:
                     bib_content = f.read()
             except Exception as e:
                 logger.error(f"Error reading uploaded file {bib_file.name}: {e}")
-                return None, f"Error reading uploaded file {bib_file.name}: {e}", f"Error reading uploaded file {bib_file.name}: {e}",""
+                return None, f"Error reading uploaded file {bib_file.name}: {e}", f"Error reading uploaded file {bib_file.name}: {e}", ""
 
             # Parse BibTeX data
             try:
@@ -466,47 +447,46 @@ class PaperDownloader:
             downloaded_files = []
             failed_dois = []
             downloaded_links = []
-           
             for i,doi in enumerate(dois):
-                  result = await self._download_single_doi(doi, cancel_event)
+                result = await self._download_single_doi(doi, cancel_event)
 
-                  if cancel_event.is_set():
+                if cancel_event.is_set():
                        logger.info("Download Cancelled in bibtex mode")
-                       return None, "Download Cancelled", "Download Cancelled", ""#cancel if requested
-                      
-                  if result is None:
-                        continue
-                  
-                  if isinstance(result, Exception):
-                    # Excepción inesperada
-                      error_msg = f"Unexpected error: {str(result)}"
-                      logger.error(f"Error downloading {doi}: {error_msg}")
-                      failed_dois.append(f'<a href="https://doi.org/{doi}">{doi}</a> - {error_msg}')
+                       return  None, "Download Cancelled", "Download Cancelled", ""#cancel if requested
+                if result is None:
+                    continue
+
+                if isinstance(result, Exception):
+                     # Excepción inesperada
+                     error_msg = f"Unexpected error: {str(result)}"
+                     logger.error(f"Error downloading {doi}: {error_msg}")
+                     failed_dois.append(f'<a href="https://doi.org/{doi}">{doi}</a> - {error_msg}')
             
-                  elif result[0] is None:
+                 elif result[0] is None:
                      # Descarga fallida (resultado de download_single_doi_async)
-                     error_msg = result[1]
-                     logger.warning(f"Failed to download {doi}: {error_msg}")
-                     failed_dois.append(f'<a href="https://doi.org/{doi}">{doi}</a> - {error_msg}')
-                
-                  else:
-                    # Descarga exitosa
+                    error_msg = result[1]
+                    logger.warning(f"Failed to download {doi}: {error_msg}")
+                    failed_dois.append(f'<a href="https://doi.org/{doi}">{doi}</a> - {error_msg}')
+            
+                 else:
+                     # Descarga exitosa
                      filepath = result[0]
-                         
+                     
                     # Unique filename for zip
-                     filename = f"{str(doi).replace('/', '_').replace('.', '_')}_{i}.pdf"
+                     filename = f"{str(doi).replace('/', '_').replace('.', '_')}_{i}.pdf" #indent fixed
                      filepath_unique = os.path.join(self.output_dir, filename)
                      os.rename(filepath, filepath_unique)
                      downloaded_files.append(filepath_unique)
                      downloaded_links.append(f'<a href="https://doi.org/{doi}">{doi}</a>')
 
+
             if downloaded_files:
                 zip_filename = 'papers.zip'
                 loop = asyncio.get_running_loop()
                 loop.run_in_executor(self.executor, lambda:  self.create_zip(zip_filename,downloaded_files))
                 logger.info(f"ZIP file created: {zip_filename}")
-                
-            return  zip_filename, "\n".join(downloaded_links), "\n".join(failed_dois), ""
+    
+            return zip_filename, "\n".join(downloaded_links), "\n".join(failed_dois), ""
 
     def create_zip(self, zip_filename, files):
         """Crea un archivo zip con los pdfs descargados"""
@@ -529,32 +509,25 @@ def create_gradio_interface():
        return  gr.Textbox.update(value=f"{message}"),gr.Textbox.update(value=f"<pre>{logs}</pre>")
     
 
-    async def download_papers(bib_file, doi_input, dois_input, output_file, downloaded_dois_textbox,failed_dois_textbox,logs, single_file):
-            cancel_event = asyncio.Event() # Create cancellation event for every submission.
-            downloader.cancel_event = cancel_event # store the event so that it is available to stop the process
-            if bib_file:
-                 # Check file type
-                if not bib_file.name.lower().endswith('.bib'):
-                     return None, "Error: Please upload a .bib file", "Error: Please upload a .bib file",  "",  None #Added for consistent value returns
-
-                zip_file, downloaded_dois, failed_dois, logs_text=  await downloader.process_bibtex(bib_file, cancel_event)
-
-                return zip_file, downloaded_dois, failed_dois, logs_text, None# Correctly send None and "", in one var to return tuple 5 values, as it is required
-
-            elif doi_input:
-
-                  filepath, message, error= await downloader._download_single_doi(doi_input, cancel_event)
-                  return None, message, error,"", filepath # Correctly return params here on singe execution
-
-            elif dois_input:
-                
-                zip_file, downloaded_dois, failed_dois, logs_text =  await downloader.download_multiple_dois(dois_input, cancel_event)
-
-                return  zip_file, downloaded_dois, failed_dois, logs_text, None # Correctly send null
-           
-            else:
-                return None, "Please provide a .bib file, a single DOI, or a list of DOIs", "Please provide a .bib file, a single DOI, or a list of DOIs","",  None # must also have five values to satisfy gradio block method
-        
+    async def download_papers(bib_file, doi_input, dois_input):
+           cancel_event = asyncio.Event() # Create cancellation event for every submission.
+           downloader.cancel_event = cancel_event # store the event so that it is available to stop the process
+           if bib_file:
+              # Check file type
+               if not bib_file.name.lower().endswith('.bib'):
+                     return None, "Error: Please upload a .bib file", "Error: Please upload a .bib file",  "", None #Fixed with default value 
+
+               zip_file, downloaded_dois, failed_dois, logs_text=  await downloader.process_bibtex(bib_file, cancel_event)
+               return zip_file, downloaded_dois, failed_dois, logs_text, None
+           elif doi_input:
+                 filepath, message, error =  await downloader._download_single_doi(doi_input, cancel_event)
+                 return None, message, error,"", filepath# correct returns.
+          
+           elif dois_input:
+                  zip_file, downloaded_dois, failed_dois, logs_text=  await downloader.download_multiple_dois(dois_input, cancel_event)
+                  return  zip_file, downloaded_dois, failed_dois, logs_text, None
+           else:
+                 return   None, "Please provide a .bib file, a single DOI, or a list of DOIs", "Please provide a .bib file, a single DOI, or a list of DOIs", "", None
     
     with gr.Blocks(theme="Hev832/Applio", css="""
         .gradio-container {
@@ -615,12 +588,11 @@ def create_gradio_interface():
             submit_button.click(
                 download_papers,
                 inputs=[bib_file, doi_input, dois_input],
-                  outputs=[output_file, downloaded_dois_textbox, failed_dois_textbox,logs, single_file ] # the new output should be a tuple and we output logs too for debugging.
+                  outputs=[output_file, downloaded_dois_textbox, failed_dois_textbox,logs, single_file ],  # the new output should be a tuple and we output logs too for debugging.
             )
-  
+    
     interface.title="🔬 Academic Paper Batch Downloader"
     interface.description="Upload a BibTeX file or enter DOIs to download PDFs. We'll attempt to fetch PDFs from multiple sources like Sci-Hub, Libgen, Google Scholar and Crossref. You can use any of the three inputs at any moment."
-   
     return interface
 
 def main():