Update app.py

app.py

@@ -128,37 +128,44 @@ def extract_full_paper_with_labels(pdf_path, progress=None):
         "content": content
     }
 
-def upload_with_progress(file_path, repo_id, token, progress):
-    """
-    Upload file to Hugging Face Dataset using upload_file() API method.
-    """
-    print(f"📤 Starting upload of Parquet: {file_path}")
-    file_size = os.path.getsize(file_path)
-
-    api = HfApi()
-
-    try:
-        # Use upload_file() method from huggingface_hub
-        api.upload_file(
-            path_or_fileobj=file_path,
-            path_in_repo=os.path.basename(file_path),
-            repo_id=repo_id,
-            repo_type="dataset",
-            token=token
-        )
+def pdf_to_parquet_and_upload(pdf_files, hf_token, dataset_repo_id, action_choice, progress=gr.Progress()):
+    upload_message = ""
 
+    total_files = len(pdf_files)
+    print("🚀 Starting PDF to Parquet Conversion Process")
+
+    for idx, pdf_file in enumerate(pdf_files):
         if progress is not None:
-            progress(1, desc="✅ Upload Complete")
+            progress(idx / total_files, desc=f"Processing File {idx + 1}/{total_files}")
+
+        # ✅ Step 1: Process PDF with Full Labels
+        extracted_data = extract_full_paper_with_labels(pdf_file.name, progress=progress)
+
+        # ✅ Step 2: Use Title for Parquet Filename
+        sanitized_title = sanitize_filename(extracted_data["title"])
+        parquet_file = f"{sanitized_title}.parquet"
+
+        # Convert to DataFrame
+        df = pd.DataFrame([extracted_data])
+
+        try:
+            df.to_parquet(parquet_file, engine='pyarrow', index=False)
+            print(f"✅ Parquet saved as: {parquet_file}")
+        except Exception as e:
+            print(f"❌ Parquet Conversion Failed: {str(e)}")
+            return None, f"❌ Parquet Conversion Failed: {str(e)}"
+
+        # ✅ Step 3: Upload Parquet (if selected)
+        if action_choice in ["Upload to Hugging Face", "Both"]:
+            try:
+                upload_message = upload_with_progress(parquet_file, dataset_repo_id, hf_token, progress)
+            except Exception as e:
+                print(f"❌ Upload Failed: {str(e)}")
+                upload_message = f"❌ Upload failed: {str(e)}"
 
-        print(f"✅ Successfully uploaded to {repo_id}")
-        return f"✅ Successfully uploaded to {repo_id}"
+    print("🏁 Process Completed")
+    return parquet_file, upload_message
 
-    except HfHubHTTPError as e:
-        print(f"❌ Upload failed: {e}")
-        return f"❌ Upload failed: {str(e)}"
-    except Exception as e:
-        print(f"❌ Unexpected error: {e}")
-        return f"❌ Unexpected error: {str(e)}"
 
 # ✅ Gradio Interface
 iface = gr.Interface(