Added additional password auth for AWS-based files. Changed 'Clean' default to no

.gitignore

@@ -17,6 +17,7 @@
 *.pkl.gz
 *.pem
 *.json.out
+*.env
 docs/*
 build/*
 dist/*
@@ -25,4 +26,5 @@ db/*
 experiments/*
 model/*
 build_deps/*
-build_deps_old/*
+build_deps_old/*
+AWS errors 19-03.txt

app.py

@@ -157,6 +157,7 @@ depends on factors such as the type of documents or queries. Information taken f
             in_join_column = gr.Dropdown(label="Column to join in new data frame")
             search_df_join_column = gr.Dropdown(label="Column to join in search data frame")
         with gr.Accordion(label = "AWS data access", open = False):
+            aws_password_box = gr.Textbox(label="Password for AWS data access (ask Data team if you don't have this)")
             with gr.Row():
                 in_aws_keyword_file = gr.Dropdown(label="Choose keyword file to load from AWS (only valid for API Gateway app)", choices=["None", "Bioasq - Biomedical example data - keyword search"])
                 load_aws_keyword_data_button = gr.Button(value="Load keyword data from AWS", variant="secondary")
@@ -176,8 +177,8 @@ depends on factors such as the type of documents or queries. Information taken f
     in_no_search_results_button.click(display_info, inputs=in_no_search_info)
 
     ### Loading AWS data ###
-    load_aws_keyword_data_button.click(fn=load_data_from_aws, inputs=[in_aws_keyword_file], outputs=[in_bm25_file, out_aws_data_message])
-    load_aws_semantic_data_button.click(fn=load_data_from_aws, inputs=[in_aws_semantic_file], outputs=[in_semantic_file, out_aws_data_message])
+    load_aws_keyword_data_button.click(fn=load_data_from_aws, inputs=[in_aws_keyword_file, aws_password_box], outputs=[in_bm25_file, out_aws_data_message])
+    load_aws_semantic_data_button.click(fn=load_data_from_aws, inputs=[in_aws_semantic_file, aws_password_box], outputs=[in_semantic_file, out_aws_data_message])
           
     
     ### BM25 SEARCH ###

search_funcs/aws_functions.py

@@ -106,60 +106,63 @@ def download_files_from_s3(bucket_name, s3_folder, local_folder, filenames):
 
 
 
-def load_data_from_aws(in_aws_keyword_file, bucket_name=bucket_name):
+def load_data_from_aws(in_aws_keyword_file, aws_password="", bucket_name=bucket_name):
 
     temp_dir = tempfile.mkdtemp()
     local_keyword_stub = temp_dir + '/keyword/'
     local_semantic_stub = temp_dir + '/semantic/'
 
     files = []
+    if aws_password:
+        if "Bioasq - Biomedical example data" in in_aws_keyword_file and aws_password == os.environ['BIOASQ_PASSWORD']:
 
-    if "Bioasq - Biomedical example data" in in_aws_keyword_file:
+            s3_folder_stub = 'example_data/bioasq/latest/'
 
-        s3_folder_stub = 'example_data/bioasq/latest/'
-
-        if 'keyword' in in_aws_keyword_file:
-            s3_folder_stub = s3_folder_stub + 'keyword/'
-            local_folder_path = local_keyword_stub
+            if 'keyword' in in_aws_keyword_file:
+                s3_folder_stub = s3_folder_stub + 'keyword/'
+                local_folder_path = local_keyword_stub
 
-        if 'semantic' in in_aws_keyword_file:
-            s3_folder_stub = s3_folder_stub + 'semantic/'
-            local_folder_path = local_semantic_stub
-                  
+            if 'semantic' in in_aws_keyword_file:
+                s3_folder_stub = s3_folder_stub + 'semantic/'
+                local_folder_path = local_semantic_stub
+                    
 
-        # Check if folder exists
-        if not os.path.exists(local_folder_path):
-            print(f"Folder {local_folder_path} does not exist! Making folder.")
+            # Check if folder exists
+            if not os.path.exists(local_folder_path):
+                print(f"Folder {local_folder_path} does not exist! Making folder.")
 
-            os.mkdir(local_folder_path)
+                os.mkdir(local_folder_path)
 
-        # Check if folder is empty
-        if len(os.listdir(local_folder_path)) == 0:
-            print(f"Folder {local_folder_path} is empty")
+            # Check if folder is empty
+            if len(os.listdir(local_folder_path)) == 0:
+                print(f"Folder {local_folder_path} is empty")
 
-            if 'keyword' in in_aws_keyword_file:
-                # Download keyword folder
-                download_files_from_s3(bucket_name, s3_folder_stub, local_folder_path, filenames='*')
+                if 'keyword' in in_aws_keyword_file:
+                    # Download keyword folder
+                    download_files_from_s3(bucket_name, s3_folder_stub, local_folder_path, filenames='*')
 
-            if 'semantic' in in_aws_keyword_file:
-                # Download keyword folder
-                download_files_from_s3(bucket_name, s3_folder_stub, local_folder_path, filenames=['mini-bioasq-0000_cleaned_bge_embedding_compress.npz', 'mini-bioasq-0000_cleaned_prepared_docs.pkl.gz'])
+                if 'semantic' in in_aws_keyword_file:
+                    # Download keyword folder
+                    download_files_from_s3(bucket_name, s3_folder_stub, local_folder_path, filenames=['mini-bioasq-0000_cleaned_bge_embedding_compress.npz', 'mini-bioasq-0000_cleaned_prepared_docs.pkl.gz'])
 
-            print("AWS data downloaded")
+                print("AWS data downloaded")
 
-        else:
-            print(f"Folder {local_folder_path} is not empty")
+            else:
+                print(f"Folder {local_folder_path} is not empty")
 
-        #files = os.listdir(local_folder_stub)
-        #print(files)
+            #files = os.listdir(local_folder_stub)
+            #print(files)
 
-        files = [os.path.join(local_folder_path, f) for f in os.listdir(local_folder_path) if os.path.isfile(os.path.join(local_folder_path, f))]
+            files = [os.path.join(local_folder_path, f) for f in os.listdir(local_folder_path) if os.path.isfile(os.path.join(local_folder_path, f))]
 
-        out_message = "Data successfully loaded from AWS"
-        print(out_message)
+            out_message = "Data successfully loaded from AWS"
+            print(out_message)
 
+        else:
+            out_message = "Data not loaded from AWS"
+            print(out_message)
     else:
-        out_message = "Data not loaded from AWS"
+        out_message = "No password provided. Please ask the data team for access if you need this."
         print(out_message)
 
     return files, out_message