Update app.py

app.py

@@ -127,27 +127,42 @@ def extract_full_paper_with_labels(pdf_path, progress=None):
 
 def upload_with_progress(file_path, repo_id, token, progress):
     """
-    Upload file to Hugging Face Dataset using the official SDK with progress tracking.
+    Upload file to Hugging Face Dataset with progress tracking.
     """
-    from huggingface_hub import HfApi
-    from huggingface_hub.utils import HfHubHTTPError
-    import os
 
     print(f"📤 Starting upload of Parquet: {file_path}")
     file_size = os.path.getsize(file_path)
-
     api = HfApi()
 
+    # Get the proper upload URL from the Hugging Face API
+    upload_url = f"https://huggingface.co/api/datasets/{repo_id}/upload"
+
     try:
-        # ✅ Official upload method from huggingface_hub
-        api.upload_file(
-            path_or_fileobj=file_path,
-            path_in_repo=os.path.basename(file_path),
-            repo_id=repo_id,
-            repo_type="dataset",  # Important to specify it's a dataset repo
-            token=token
-        )
+        with open(file_path, 'rb') as f:
+            chunk_size = 1024 * 1024  # 1 MB chunks
+            uploaded = 0
+
+            headers = {
+                "Authorization": f"Bearer {token}",
+                "Content-Type": "application/octet-stream"
+            }
+
+            while True:
+                chunk = f.read(chunk_size)
+                if not chunk:
+                    break  # Finished reading file
 
+                response = requests.put(upload_url, headers=headers, data=chunk)
+
+                if response.status_code != 200:
+                    raise Exception(f"Upload failed: {response.text}")
+
+                # Update progress after each chunk
+                uploaded += len(chunk)
+                if progress is not None:
+                    progress(uploaded / file_size, desc=f"Uploading... {uploaded // (1024 * 1024)}MB/{file_size // (1024 * 1024)}MB")
+
+        # Final progress update
         if progress is not None:
             progress(1, desc="✅ Upload Complete")
 
@@ -162,6 +177,7 @@ def upload_with_progress(file_path, repo_id, token, progress):
         return f"❌ Unexpected error: {str(e)}"
 
 
+
 def pdf_to_parquet_and_upload(pdf_files, hf_token, dataset_repo_id, action_choice, progress=gr.Progress()):
     upload_message = ""