Made changes to hopefully resolve issue with downloading cost centre details from S3 to container

app.py

@@ -648,7 +648,7 @@ with app:
 
     # If relevant environment variable is set, load in the default allow list file from S3 or locally. Even when setting S3 path, need to local path to give a download location
     if GET_DEFAULT_ALLOW_LIST == "True" and (ALLOW_LIST_PATH or S3_ALLOW_LIST_PATH):
-        if not os.path.exists(ALLOW_LIST_PATH) and S3_ALLOW_LIST_PATH:
+        if not os.path.exists(ALLOW_LIST_PATH) and S3_ALLOW_LIST_PATH and RUN_AWS_FUNCTIONS == 1:
             app.load(download_file_from_s3, inputs=[s3_default_bucket, s3_default_allow_list_file, default_allow_list_output_folder_location]).\
             success(load_in_default_allow_list, inputs = [default_allow_list_output_folder_location], outputs=[in_allow_list])
             print("Successfully loaded allow list from S3")
@@ -659,7 +659,7 @@ with app:
 
     # If relevant environment variable is set, load in the default cost code file from S3 or locally
     if GET_COST_CODES == "True" and (COST_CODES_PATH or S3_COST_CODES_PATH):  
-        if not os.path.exists(COST_CODES_PATH) and S3_COST_CODES_PATH:
+        if not os.path.exists(COST_CODES_PATH) and S3_COST_CODES_PATH and RUN_AWS_FUNCTIONS == 1:
             app.load(download_file_from_s3, inputs=[s3_default_bucket, s3_default_cost_codes_file, default_cost_codes_output_folder_location]).\
             success(load_in_default_cost_codes, inputs = [default_cost_codes_output_folder_location, default_cost_code_textbox], outputs=[cost_code_dataframe, cost_code_dataframe_base, cost_code_choice_drop])
             print("Successfully loaded cost codes from S3")

tools/aws_functions.py

@@ -40,9 +40,20 @@ if RUN_AWS_FUNCTIONS == "1":
 def download_file_from_s3(bucket_name:str, key:str, local_file_path_and_name:str, RUN_AWS_FUNCTIONS:str = RUN_AWS_FUNCTIONS):
 
     if RUN_AWS_FUNCTIONS == "1":
-        s3 = boto3.client('s3', region_name=AWS_REGION)
-        s3.download_file(bucket_name, key, local_file_path_and_name)
-        print(f"File downloaded from s3://{bucket_name}/{key} to {local_file_path_and_name}")
+
+        try:
+            print("bucket_name:", bucket_name)
+            print("key:", key)
+            print("local_file_path_and_name:", local_file_path_and_name)
+
+            # Ensure the local directory exists
+            os.makedirs(os.path.dirname(local_file_path_and_name), exist_ok=True)
+
+            s3 = boto3.client('s3', region_name=AWS_REGION)
+            s3.download_file(bucket_name, key, local_file_path_and_name)
+            print(f"File downloaded from s3://{bucket_name}/{key} to {local_file_path_and_name}")
+        except Exception as e:
+            print("Could not download file:", key, "from s3 due to", e)
 
                          
 def download_folder_from_s3(bucket_name:str, s3_folder:str, local_folder:str, RUN_AWS_FUNCTIONS:str = RUN_AWS_FUNCTIONS):

tools/config.py

@@ -59,6 +59,8 @@ def add_folder_to_path(folder_path: str):
     else:
         print(f"Folder not found at {folder_path} - not added to PATH")
 
+ensure_folder_exists("config/")
+
 # If you have an aws_config env file in the config folder, you can load in app variables this way, e.g. 'config/app_config.env'
 APP_CONFIG_PATH = get_or_create_env_var('APP_CONFIG_PATH', 'config/app_config.env') # e.g. config/app_config.env
 
@@ -204,7 +206,10 @@ REDACTION_LANGUAGE = get_or_create_env_var("REDACTION_LANGUAGE", "en") # Current
 ###
 
 TLDEXTRACT_CACHE = get_or_create_env_var('TLDEXTRACT_CACHE', 'tld/.tld_set_snapshot')
-extract = TLDExtract(cache_dir=TLDEXTRACT_CACHE)
+try:
+    extract = TLDExtract(cache_dir=TLDEXTRACT_CACHE)
+except:
+    extract = TLDExtract(cache_dir=None)
 
 # Get some environment variables and Launch the Gradio app
 COGNITO_AUTH = get_or_create_env_var('COGNITO_AUTH', '0')
@@ -232,7 +237,7 @@ else: OUTPUT_ALLOW_LIST_PATH = 'config/default_allow_list.csv'
 
 SHOW_COSTS = get_or_create_env_var('SHOW_COSTS', 'False')
 
-GET_COST_CODES = get_or_create_env_var('GET_COST_CODES', 'False')
+GET_COST_CODES = get_or_create_env_var('GET_COST_CODES', 'True')
 
 DEFAULT_COST_CODE = get_or_create_env_var('DEFAULT_COST_CODE', '')