Update app.py

app.py

@@ -12,6 +12,8 @@ from bs4 import BeautifulSoup
 import io
 import asyncio
 import aiohttp
+from concurrent.futures import ThreadPoolExecutor
+
 
 # Configure logging
 logging.basicConfig(level=logging.INFO,
@@ -41,6 +43,7 @@ class PaperDownloader:
             'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
             'Accept-Language': 'en-US,en;q=0.9',
         }
+        self.executor = ThreadPoolExecutor(max_workers=4)
 
     def clean_doi(self, doi):
         """Clean and encode DOI for URL"""
@@ -59,47 +62,44 @@ class PaperDownloader:
                 return None, None
 
     async def fetch_pdf_content(self, session, url, max_redirects=5, max_retries=2, retry_delay=1):
-            """Fetch content and validate if response is PDF, following up to max_redirects redirections with retries."""
-            
-            current_url = url
-            redirect_count = 0
-            retry_count = 0
-    
-            while redirect_count <= max_redirects:
-                try:
-                    while retry_count <= max_retries:
-                        try:
-                            async with session.get(current_url, headers=self.headers, timeout=10, allow_redirects=False) as response:
-    
-                                if response.status in [301, 302, 307, 308]:
-                                    current_url = response.headers['Location']
-                                    redirect_count += 1
-                                    logger.debug(f"Following redirect from {url} to {current_url}")
-                                    break  # Break out of the retry loop for a redirect
-    
-                                response.raise_for_status()
-    
-                                if 'application/pdf' in response.headers.get('Content-Type', ''):
-                                    return await response.read()
-                                else:
-                                    logger.debug(
-                                        f"Content type not PDF for {current_url}: {response.headers.get('Content-Type', '')}")
-                                    return None
-                        except Exception as e:
-                            logger.debug(
-                                f"Error getting PDF, retrying ({retry_count}/{max_retries}) from {current_url}: {e}")
-                            retry_count += 1
-                            await asyncio.sleep(retry_delay)
-    
-                    retry_count = 0  # Reset the retry count, in case there's a next redirect attempt
-    
-                except Exception as e:
-                    logger.debug(f"Error getting PDF from {current_url}: {e}")
-                    return None
-    
-            logger.debug(f"Too many redirects or retries {url}, not following this link further")
-            return None
-    
+        """Fetch content and validate if response is PDF, following up to max_redirects redirections with retries."""
+        
+        current_url = url
+        redirect_count = 0
+        retry_count = 0
+
+        while redirect_count <= max_redirects:
+            try:
+                while retry_count <= max_retries:
+                    try:
+                        async with session.get(current_url, headers=self.headers, timeout=10, allow_redirects=False) as response:
+                            if response.status in [301, 302, 307, 308]:
+                                current_url = response.headers['Location']
+                                redirect_count += 1
+                                logger.debug(f"Following redirect from {url} to {current_url}")
+                                break  # Break out of the retry loop for a redirect
+
+                            response.raise_for_status()
+
+                            if 'application/pdf' in response.headers.get('Content-Type', ''):
+                                return await response.read()
+                            else:
+                                logger.debug(f"Content type not PDF for {current_url}: {response.headers.get('Content-Type', '')}")
+                                return None
+                    except Exception as e:
+                         logger.debug(f"Error getting PDF, retrying ({retry_count}/{max_retries}) from {current_url}: {e}")
+                         retry_count += 1
+                         await asyncio.sleep(retry_delay)
+
+                retry_count = 0  # Reset the retry count, in case there's a next redirect attempt
+
+            except Exception as e:
+                logger.debug(f"Error getting PDF from {current_url}: {e}")
+                return None
+
+        logger.debug(f"Too many redirects or retries {url}, not following this link further")
+        return None
+        
     async def download_paper_direct_doi_async(self, session, doi):
             """Attempt to download the pdf from the landing page of the doi"""
             if not doi:
@@ -124,7 +124,7 @@ class PaperDownloader:
                     r'(https?://[^\s<>"]+?download/[^\s<>"]+)',
                     r'(https?://[^\s<>"]+?\/pdf\/[^\s<>"]+)',
                     ]
-    
+                 
                 pdf_urls = []
                 for pattern in pdf_patterns:
                     pdf_urls.extend(re.findall(pattern, text))
@@ -177,34 +177,34 @@ class PaperDownloader:
         return None
 
     async def download_paper_libgen_async(self, session, doi):
-            """Download from Libgen, handles the query and the redirection"""
-            if not doi:
-                return None
-    
-            base_url = 'https://libgen.rs/scimag/'
-            try:
-                search_url = f"{base_url}?q={self.clean_doi(doi)}"
-                text, headers = await self.fetch_with_headers(session, search_url, timeout=10)
-    
-                if not text or "No results" in text:
-                    logger.debug(f"No results for DOI: {doi} on libgen")
-                    return None
-    
-                soup = BeautifulSoup(text, 'html.parser')
-    
-                links = soup.select('table.c > tbody > tr:nth-child(2) > td:nth-child(1) > a')
-    
-                if links:
-                    link = links[0]
-                    pdf_url = link['href']
-                    pdf_content = await self.fetch_pdf_content(session, pdf_url)
-                    if pdf_content:
-                        logger.debug(f"Found PDF from: {pdf_url}")
-                        return pdf_content
-            except Exception as e:
-                logger.debug(f"Error trying to download {doi} from libgen: {e}")
+        """Download from Libgen, handles the query and the redirection"""
+        if not doi:
             return None
-    
+
+        base_url = 'https://libgen.rs/scimag/'
+        try:
+            search_url = f"{base_url}?q={self.clean_doi(doi)}"
+            text, headers = await self.fetch_with_headers(session, search_url, timeout=10)
+
+            if not text or "No results" in text:
+                logger.debug(f"No results for DOI: {doi} on libgen")
+                return None
+
+            soup = BeautifulSoup(text, 'html.parser')
+
+            links = soup.select('table.c > tbody > tr:nth-child(2) > td:nth-child(1) > a')
+
+            if links:
+                link = links[0]
+                pdf_url = link['href']
+                pdf_content = await self.fetch_pdf_content(session, pdf_url)
+                if pdf_content:
+                    logger.debug(f"Found PDF from: {pdf_url}")
+                    return pdf_content
+        except Exception as e:
+            logger.debug(f"Error trying to download {doi} from libgen: {e}")
+        return None
+
     async def download_paper_google_scholar_async(self, session, doi):
         """Search google scholar to find an article with the given doi, try to get the pdf"""
         if not doi:
@@ -230,39 +230,40 @@ class PaperDownloader:
                 if pdf_content:
                   logger.debug(f"Found PDF from: {pdf_url}")
                   return pdf_content
+
         except Exception as e:
             logger.debug(f"Google Scholar error for {doi}: {e}")
 
         return None
     
     async def download_paper_crossref_async(self, session, doi):
-        """Alternative search method using Crossref"""
-        if not doi:
-            return None
-    
-        try:
-            # Search for open access link
-            url = f"https://api.crossref.org/works/{doi}"
-            response = await session.get(url, headers=self.headers, timeout=10)
-    
-            if response.status == 200:
-                data = await response.json()
-                work = data.get('message', {})
-    
-                # Search for open access links
-                links = work.get('link', [])
-                for link in links:
-                    if link.get('content-type') == 'application/pdf':
-                        pdf_url = link.get('URL')
-                        if pdf_url:
-                            pdf_content = await self.fetch_pdf_content(session, pdf_url)
-                            if pdf_content:
-                                logger.debug(f"Found PDF from: {pdf_url}")
-                                return pdf_content
-        except Exception as e:
-            logger.debug(f"Crossref error for {doi}: {e}")
-        return None
+            """Alternative search method using Crossref"""
+            if not doi:
+                return None
     
+            try:
+                # Search for open access link
+                url = f"https://api.crossref.org/works/{doi}"
+                response = await session.get(url, headers=self.headers, timeout=10)
+    
+                if response.status == 200:
+                    data = await response.json()
+                    work = data.get('message', {})
+    
+                    # Search for open access links
+                    links = work.get('link', [])
+                    for link in links:
+                        if link.get('content-type') == 'application/pdf':
+                            pdf_url = link.get('URL')
+                            if pdf_url:
+                                pdf_content = await self.fetch_pdf_content(session, pdf_url)
+                                if pdf_content:
+                                    logger.debug(f"Found PDF from: {pdf_url}")
+                                    return pdf_content
+            except Exception as e:
+                logger.debug(f"Crossref error for {doi}: {e}")
+            return None
+
     async def download_with_retry_async(self, doi, max_retries=3, initial_delay=2):
         """Downloads a paper using multiple strategies with exponential backoff and async requests"""
         pdf_content = None
@@ -283,7 +284,7 @@ class PaperDownloader:
                         return pdf_content
                 except Exception as e:
                     logger.error(f"Error in download attempt {retries + 1} for DOI {doi}: {e}")
-    
+
                 if not pdf_content:
                     retries += 1
                     logger.warning(f"Retry attempt {retries} for DOI: {doi} after {delay} seconds")
@@ -292,36 +293,38 @@ class PaperDownloader:
     
         return None
     
-    async def download_single_doi_async(self, doi):
-        """Downloads a single paper using a DOI"""
-        if not doi:
-            return None, "Error: DOI not provided", "Error: DOI not provided"
-
-        try:
-            pdf_content = await self.download_with_retry_async(doi)
-
-            if pdf_content:
-                if doi is None:
-                     return None, "Error: DOI not provided", "Error: DOI not provided"
-                filename = f"{str(doi).replace('/', '_').replace('.', '_')}.pdf"
-                filepath = os.path.join(self.output_dir, filename)
-
-                # Write file asynchronously
-                loop = asyncio.get_running_loop()
-                await loop.run_in_executor(None, lambda: open(filepath, 'wb').write(pdf_content))
-
-                logger.info(f"Successfully downloaded: {filename}")
-                return filepath, f'<a href="https://doi.org/{doi}">{doi}</a>', ""
-            else:
-                 logger.warning(f"Could not download: {doi}")
-                 return None, f"Could not download {doi}", f'<a href="https://doi.org/{doi}">{doi}</a>'
+    async def download_single_doi_async(self, doi, progress_callback):
+            """Downloads a single paper using a DOI, and updates the given progress_callback"""
+            if not doi:
+                return None, "Error: DOI not provided", "Error: DOI not provided"
+    
+            try:
+                pdf_content = await self.download_with_retry_async(doi)
+    
+                if pdf_content:
+                    if doi is None:
+                        return None, "Error: DOI not provided", "Error: DOI not provided"
+                    filename = f"{str(doi).replace('/', '_').replace('.', '_')}.pdf"
+                    filepath = os.path.join(self.output_dir, filename)
 
-        except Exception as e:
-             logger.error(f"Error processing {doi}: {e}")
-             return None, f"Error processing {doi}: {e}", f"Error processing {doi}: {e}"
-        
-    async def download_multiple_dois_async(self, dois_text):
-            """Downloads multiple papers from a list of DOIs"""
+                    loop = asyncio.get_running_loop()
+                    await loop.run_in_executor(self.executor, lambda: open(filepath, 'wb').write(pdf_content))
+                    
+                    logger.info(f"Successfully downloaded: {filename}")
+                    progress_callback(f"Successfully downloaded: <a href='https://doi.org/{doi}'>{doi}</a>")
+                    return filepath, f'<a href="https://doi.org/{doi}">{doi}</a>', ""
+                else:
+                    logger.warning(f"Could not download: {doi}")
+                    progress_callback(f"Could not download:  <a href='https://doi.org/{doi}'>{doi}</a>")
+                    return None, f"Could not download {doi}", f'<a href="https://doi.org/{doi}">{doi}</a>'
+    
+            except Exception as e:
+                logger.error(f"Error processing {doi}: {e}")
+                progress_callback(f"Error processing {doi}:  <a href='https://doi.org/{doi}'>{doi}</a> {e}")
+                return None, f"Error processing {doi}: {e}", f"Error processing {doi}: {e}"
+    
+    async def download_multiple_dois_async(self, dois_text, progress_callback):
+            """Downloads multiple papers from a list of DOIs and uses a callback for UI"""
             if not dois_text:
                 return None, "Error: No DOIs provided", "Error: No DOIs provided"
     
@@ -332,34 +335,34 @@ class PaperDownloader:
             downloaded_files = []
             failed_dois = []
             downloaded_links = []
-            
-            for i, doi in enumerate(tqdm(dois, desc="Downloading papers")):
-               filepath, success_message, fail_message = await self.download_single_doi_async(doi)
-               if filepath:
+           
+            for i, doi in enumerate(dois):
+                 filepath, success_message, fail_message = await self.download_single_doi_async(doi, progress_callback)
+                 if filepath:
                     # Unique filename for zip
                     filename = f"{str(doi).replace('/', '_').replace('.', '_')}_{i}.pdf"
                     filepath_unique = os.path.join(self.output_dir, filename)
                     os.rename(filepath, filepath_unique)
                     downloaded_files.append(filepath_unique)
                     downloaded_links.append(f'<a href="https://doi.org/{doi}">{doi}</a>')
-               else:
-                    failed_dois.append(f'<a href="https://doi.org/{doi}">{doi}</a>')
-                    
+                 else:
+                     failed_dois.append(f'<a href="https://doi.org/{doi}">{doi}</a>')
+            
             if downloaded_files:
-                    zip_filename = 'papers.zip'
-                    loop = asyncio.get_running_loop()
-                    await loop.run_in_executor(None, lambda:  self.create_zip(zip_filename,downloaded_files) )
-                    logger.info(f"ZIP file created: {zip_filename}")
-
+                zip_filename = 'papers.zip'
+                loop = asyncio.get_running_loop()
+                await loop.run_in_executor(self.executor, lambda:  self.create_zip(zip_filename,downloaded_files))
+                logger.info(f"ZIP file created: {zip_filename}")
+            
             return zip_filename if downloaded_files else None, "\n".join(downloaded_links), "\n".join(failed_dois)
     
     def create_zip(self, zip_filename, downloaded_files):
-            with zipfile.ZipFile(zip_filename, 'w') as zipf:
+        with zipfile.ZipFile(zip_filename, 'w') as zipf:
                 for file_path in downloaded_files:
                     zipf.write(file_path, arcname=os.path.basename(file_path))
 
-    async def process_bibtex_async(self, bib_file):
-            """Process BibTeX file and download papers with multiple strategies"""
+    async def process_bibtex_async(self, bib_file, progress_callback):
+            """Process BibTeX file and download papers with multiple strategies and reports UI updates using a callback"""
             # Read BibTeX file content from the uploaded object
             try:
                 with open(bib_file.name, 'r', encoding='utf-8') as f:
@@ -383,50 +386,51 @@ class PaperDownloader:
             downloaded_files = []
             failed_dois = []
             downloaded_links = []
-            
-            tasks = [self.download_single_doi_async(doi) for doi in dois]
+    
+            tasks = [self.download_single_doi_async(doi, progress_callback) for doi in dois]
             results = await asyncio.gather(*tasks)
 
-
             for i, (filepath, success_message, fail_message) in enumerate(results):
-               if filepath:
-                    # Unique filename for zip
-                    filename = f"{str(dois[i]).replace('/', '_').replace('.', '_')}_{i}.pdf"
-                    filepath_unique = os.path.join(self.output_dir, filename)
-                    os.rename(filepath, filepath_unique)
-                    downloaded_files.append(filepath_unique)
-                    downloaded_links.append(f'<a href="https://doi.org/{dois[i]}">{dois[i]}</a>')
-               else:
-                    failed_dois.append(f'<a href="https://doi.org/{dois[i]}">{dois[i]}</a>')
+                    if filepath:
+                        # Unique filename for zip
+                        filename = f"{str(dois[i]).replace('/', '_').replace('.', '_')}_{i}.pdf"
+                        filepath_unique = os.path.join(self.output_dir, filename)
+                        os.rename(filepath, filepath_unique)
+                        downloaded_files.append(filepath_unique)
+                        downloaded_links.append(f'<a href="https://doi.org/{dois[i]}">{dois[i]}</a>')
+                    else:
+                        failed_dois.append(f'<a href="https://doi.org/{dois[i]}">{dois[i]}</a>')
             
             if downloaded_files:
-                    zip_filename = 'papers.zip'
-                    loop = asyncio.get_running_loop()
-                    await loop.run_in_executor(None, lambda:  self.create_zip(zip_filename,downloaded_files) )
-                    logger.info(f"ZIP file created: {zip_filename}")
-
+                zip_filename = 'papers.zip'
+                loop = asyncio.get_running_loop()
+                await loop.run_in_executor(self.executor, lambda:  self.create_zip(zip_filename,downloaded_files))
+                logger.info(f"ZIP file created: {zip_filename}")
+    
             return zip_filename, "\n".join(downloaded_links), "\n".join(failed_dois)
 
 def create_gradio_interface():
     """Create Gradio interface for Paper Downloader"""
     downloader = PaperDownloader()
 
-    async def download_papers(bib_file, doi_input, dois_input):
-        if bib_file:
-            # Check file type
-            if not bib_file.name.lower().endswith('.bib'):
-                return None, "Error: Please upload a .bib file", "Error: Please upload a .bib file", None
-
-            zip_path, downloaded_dois, failed_dois = await downloader.process_bibtex_async(bib_file)
-            return zip_path, downloaded_dois, failed_dois, None
-        elif doi_input:
-            filepath, message, failed_doi = await downloader.download_single_doi_async(doi_input)
-            return None, message, failed_doi, filepath
-        elif dois_input:
-            zip_path, downloaded_dois, failed_dois = await downloader.download_multiple_dois_async(dois_input)
-            return zip_path, downloaded_dois, failed_dois, None
-        else:
-            return None, "Please provide a .bib file, a single DOI, or a list of DOIs", "Please provide a .bib file, a single DOI, or a list of DOIs", None
+    async def download_papers(bib_file, doi_input, dois_input, progress=gr.Progress()):
+            if bib_file:
+                # Check file type
+                if not bib_file.name.lower().endswith('.bib'):
+                    return None, "Error: Please upload a .bib file", "Error: Please upload a .bib file", None
+                
+                
+                zip_path, downloaded_dois, failed_dois = await downloader.process_bibtex_async(bib_file, progress.update)
+
+                return zip_path, downloaded_dois, failed_dois, None
+            elif doi_input:
+                filepath, message, failed_doi = await downloader.download_single_doi_async(doi_input,progress.update)
+                return None, message, failed_doi, filepath
+            elif dois_input:
+                 zip_path, downloaded_dois, failed_dois = await downloader.download_multiple_dois_async(dois_input, progress.update)
+                 return zip_path, downloaded_dois, failed_dois, None
+            else:
+                 return None, "Please provide a .bib file, a single DOI, or a list of DOIs", "Please provide a .bib file, a single DOI, or a list of DOIs", None
 
 
     # Gradio Interface
@@ -436,6 +440,7 @@ def create_gradio_interface():
             gr.File(file_types=['.bib'], label="Upload BibTeX File"),
             gr.Textbox(label="Enter Single DOI", placeholder="10.xxxx/xxxx"),
             gr.Textbox(label="Enter Multiple DOIs (one per line)", placeholder="10.xxxx/xxxx\n10.yyyy/yyyy\n...")
+           
         ],
         outputs=[
             gr.File(label="Download Papers (ZIP) or Single PDF"),