Update main.py

main.py

@@ -30,7 +30,8 @@ EMBEDDING_MODEL_NAME = "allenai-specter"
 LLM_MODEL_NAME = "Qwen/Qwen2-7B-Instruct"
 
 # Load the dataset and convert to pandas
-data = datasets.load_dataset("ccm/publications")["train"].to_pandas()
+# data = datasets.load_dataset("ccm/publications")["train"].to_pandas()
+data = pandas.read_parquet("hf://datasets/ccm/publications/data/train-00000-of-00001.parquet")
 
 # Filter out any publications without an abstract
 abstract_is_null = [