Allowed for custom output folder. Upgraded Gradio version

AddressMatcher_0.1_f.spec

@@ -0,0 +1,52 @@
+# -*- mode: python ; coding: utf-8 -*-
+from PyInstaller.utils.hooks import collect_data_files
+
+datas = []
+datas += collect_data_files('gradio_client')
+datas += collect_data_files('gradio')
+
+
+a = Analysis(
+    ['app.py'],
+    pathex=[],
+    binaries=[],
+    datas=datas,
+    hiddenimports=['pyarrow.vendored.version'],
+    hookspath=['build_deps\\'],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    noarchive=False,
+    optimize=0,
+    module_collection_mode={
+        'gradio': 'py',  # Collect gradio package as source .py files
+    }
+)
+pyz = PYZ(a.pure)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name='AddressMatcher_0.1_f',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.datas,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    name='AddressMatcher_0.1_f',
+)

Dockerfile

@@ -6,7 +6,7 @@ COPY requirements.txt .
 
 RUN pip install --no-cache-dir -r requirements.txt
 
-RUN pip install --no-cache-dir gradio==4.31.0
+RUN pip install --no-cache-dir gradio==4.32.2
 
 # Set up a new user named "user" with user ID 1000
 RUN useradd -m -u 1000 user

README.md

@@ -4,7 +4,7 @@ emoji: 🌍
 colorFrom: purple
 colorTo: gray
 sdk: gradio
-sdk_version: 4.31.0
+sdk_version: 4.32.2
 app_file: app.py
 pinned: false
 license: apache-2.0

app.py

@@ -7,6 +7,7 @@ import pandas as pd
 from tools.matcher_funcs import run_matcher
 from tools.gradio import initial_data_load, ensure_output_folder_exists
 from tools.aws_functions import load_data_from_aws
+from tools.constants import output_folder
 
 import warnings
 # Remove warnings from print statements
@@ -20,7 +21,9 @@ today_rev = datetime.now().strftime("%Y%m%d")
 
 # Base folder is where the code file is stored
 base_folder = Path(os.getcwd())
-output_folder = "output/"
+# output_folder = "output/" # This is now defined in constants
+
+
 
 ensure_output_folder_exists(output_folder)
 

how_to_create_exe_dist.txt

@@ -16,6 +16,8 @@ NOTE: for ensuring that spaCy models are loaded into the program correctly in re
 
 a) In command line: pyi-makespec --additional-hooks-dir="build_deps\\" --collect-data=gradio_client --collect-data=gradio --hidden-import pyarrow.vendored.version --onefile --name AddressMatcher_0.1 app.py
 
+pyi-makespec --additional-hooks-dir="build_deps\\" --collect-data=gradio_client --collect-data=gradio --hidden-import pyarrow.vendored.version --name AddressMatcher_0.1_f app.py
+
 b) Open the created spec file in Notepad. Add the following to the end of the Analysis section then save:
 
 a = Analysis(
@@ -27,6 +29,8 @@ a = Analysis(
 
 c) Back in command line, run this: pyinstaller --clean --noconfirm AddressMatcher_0.1.spec
 
+pyinstaller --clean --noconfirm AddressMatcher_0.1_f.spec
+
 
 9. A 'dist' folder will be created with the executable inside along with all dependencies('dist\data_text_search'). 
 

tools/addressbase_api_funcs.py

@@ -156,9 +156,6 @@ def places_api_query(query, api_key, query_type):
         print("No API key provided.")
         return pd.DataFrame() # Return blank dataframe
     
-    #print('RESPONSE:', concat_results)
-    
-    
     # Convert 'results' to DataFrame
 
     # Check if 'LPI' sub-branch exists in the JSON response
@@ -177,18 +174,7 @@ def places_api_query(query, api_key, query_type):
     if isinstance(df, pd.Series):
         print("This is a series!")
         df = df.to_frame().T  # Convert the Series to a DataFrame with a single row
-    # if isinstance(df, pd.DataFrame):
-    #     print("This is a dataframe!")
-    # else:
-    #     print("This is not a dataframe!")
-    #     return pd.DataFrame() # Return blank dataframe
-
 
-    print(df)
-    #print(df.columns)
-    #df.to_csv(query + ".csv")
-
-    
 
     overall_toc = time.perf_counter()
     time_out = f"The API call took {overall_toc - overall_tic:0.1f} seconds"

tools/aws_functions.py

@@ -13,16 +13,6 @@ except Exception as e:
     bucket_name = ''
     print(e)
 
-# sts = session.client("sts")
-# Create a Session with the IAM role ARN
-# aws_role = os.environ['AWS_ROLE_DATA_TEXT_SEARCH']
-# response = sts.assume_role(
-#     RoleArn=aws_role,
-#     RoleSessionName="ecs-test-session"
-# )
-# print(response)
-
-
 def get_assumed_role_info():
     sts = boto3.client('sts', region_name='eu-west-2', endpoint_url='https://sts.eu-west-2.amazonaws.com')
     response = sts.get_caller_identity()

tools/constants.py

@@ -11,6 +11,24 @@ from .pytorch_models import *
 PandasDataFrame = Type[pd.DataFrame]
 PandasSeries = Type[pd.Series]
 
+def get_or_create_env_var(var_name, default_value):
+    # Get the environment variable if it exists
+    value = os.environ.get(var_name)
+    
+    # If it doesn't exist, set it to the default value
+    if value is None:
+        os.environ[var_name] = default_value
+        value = default_value
+    
+    return value
+
+# Retrieving or setting output folder
+env_var_name = 'GRADIO_OUTPUT_FOLDER'
+default_value = 'output/'
+
+output_folder = get_or_create_env_var(env_var_name, default_value)
+print(f'The value of {env_var_name} is {output_folder}')
+
 # +
 ''' Fuzzywuzzy/Rapidfuzz scorer to use. Options are: ratio, partial_ratio, token_sort_ratio, partial_token_sort_ratio,
 token_set_ratio, partial_token_set_ratio, QRatio, UQRatio, WRatio (default), UWRatio
@@ -18,17 +36,11 @@ details here: https://stackoverflow.com/questions/31806695/when-to-use-which-fuz
 
 fuzzy_scorer_used = "token_set_ratio"
 
-# +
 fuzzy_match_limit = 85
-
 fuzzy_search_addr_limit = 20
-
 filter_to_lambeth_pcodes= True 
-# -
-
 standardise = False
 
-# +
 if standardise == True:
     std = "_std"
 if standardise == False:
@@ -40,8 +52,7 @@ suffix_used = dataset_name + "_" + fuzzy_scorer_used
 
 # https://stackoverflow.com/questions/59221557/tensorflow-v2-replacement-for-tf-contrib-predictor-from-saved-model
 
-ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
-print(ROOT_DIR)
+
 
 # Uncomment these lines for the tensorflow model
 #model_type = "tf"
@@ -66,30 +77,32 @@ device = "cpu"
 global labels_list
 labels_list = []
 
+ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
+
+# If in a non-standard location (e.g. on AWS Lambda Function URL, then save model to tmp drive)
+if output_folder == "output/":
+    out_model_dir = ROOT_DIR
+    print(out_model_dir)
+else:
+    out_model_dir = output_folder[:-1]
+    print(out_model_dir)
+
 model_dir_name = os.path.join(ROOT_DIR, "nnet_model" , model_stub , model_version)
-print(model_dir_name)
 
 model_path = os.path.join(model_dir_name, "saved_model.zip")
-print("model path: ")
-print(model_path)
+print("Model zip path: ", model_path)
 
 if os.path.exists(model_path):
 
     os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # Better to go without GPU to avoid 'out of memory' issues
     device = "cpu"
-        
-
-
+   
     ## The labels_list object defines the structure of the prediction outputs. It must be the same as what the model was originally trained on
-    
-
-    
+ 
     ''' Load pre-trained model '''
     
-
-
     with zipfile.ZipFile(model_path,"r") as zip_ref:
-        zip_ref.extractall(model_dir_name)
+        zip_ref.extractall(out_model_dir)
         
     # if model_stub == "addr_model_out_lon":
 
@@ -143,16 +156,15 @@ if os.path.exists(model_path):
         'Postcode',  # 14
         'IGNORE'
         ]
-        
-    #labels_list.to_csv("labels_list.csv", index = None)    
+          
             
         if (model_type == "transformer") | (model_type == "gru") | (model_type == "lstm") :   
             # Load vocab and word_to_index
-            with open(model_dir_name + "vocab.txt", "r") as f:
+            with open(out_model_dir + "/vocab.txt", "r") as f:
                 vocab = eval(f.read())
-            with open(model_dir_name + "/word_to_index.txt", "r") as f:
+            with open(out_model_dir + "/word_to_index.txt", "r") as f:
                 word_to_index = eval(f.read())
-            with open(model_dir_name + "/cat_to_idx.txt", "r") as f:
+            with open(out_model_dir + "/cat_to_idx.txt", "r") as f:
                 cat_to_idx = eval(f.read())
 
             VOCAB_SIZE = len(word_to_index)
@@ -180,8 +192,12 @@ if os.path.exists(model_path):
                 exported_model = LSTMTextClassifier(VOCAB_SIZE, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM, N_LAYERS, DROPOUT, PAD_TOKEN)
         
         
-            exported_model.load_state_dict(torch.load(model_dir_name + "output_model_" + str(data_sample_size) +\
-               "_" + str(N_EPOCHS) + "_" + model_type + ".pth", map_location=torch.device('cpu')))
+            out_model_file_name = "output_model_" + str(data_sample_size) +\
+               "_" + str(N_EPOCHS) + "_" + model_type + ".pth"
+
+            out_model_path = os.path.join(out_model_dir, out_model_file_name)
+            print("Model location: ", out_model_path)
+            exported_model.load_state_dict(torch.load(out_model_path, map_location=torch.device('cpu')))
             exported_model.eval()
 
             device='cpu'
@@ -196,13 +212,7 @@ if os.path.exists(model_path):
 
 else: exported_model = []
 
-#if exported_model:
-#        exported_model = exported_model
-#else: exported_model = []
-
-
-
-# +
+### ADDRESS MATCHING FUNCTIONS
 # Address matcher will try to match <batch_size> records in one go to avoid exceeding memory limits.
 batch_size = 10000
 ref_batch_size = 150000
@@ -215,7 +225,6 @@ ref_batch_size = 150000
 
  Comparison of some of the Jellyfish string comparison methods: https://manpages.debian.org/testing/python-jellyfish-doc/jellyfish.3.en.html '''
 
-
 fuzzy_method = "jarowinkler"
 
 # Required overall match score for all columns to count as a match

tools/gradio.py

@@ -60,9 +60,9 @@ def ensure_output_folder_exists(output_folder):
     if not os.path.exists(folder_name):
         # Create the folder if it doesn't exist
         os.makedirs(folder_name)
-        print(f"Created the output folder.")
+        print(f"Created the output folder:", folder_name)
     else:
-        print(f"The output folder already exists.")
+        print(f"The output folder already exists:", folder_name)
 
 def dummy_function(in_colnames):
     """
@@ -72,4 +72,5 @@ def dummy_function(in_colnames):
 
 
 def clear_inputs(in_file, in_ref, in_text):
-    return gr.File.update(value=[]), gr.File.update(value=[]), gr.Textbox.update(value='')
+    return gr.File(value=[]), gr.File(value=[]), gr.Textbox(value='')
+

tools/matcher_funcs.py

@@ -169,7 +169,7 @@ def run_all_api_calls(in_api_key:str, Matcher:MatcherClass, query_type:str, prog
             if (i + 1) % 500 == 0:
                 print("Saving api call checkpoint for query:", str(i + 1))
                 
-                pd.concat(loop_list).to_parquet(api_ref_save_loc + ".parquet", index=False)
+                pd.concat(loop_list).to_parquet(output_folder + api_ref_save_loc + ".parquet", index=False)
 
             return loop_list
         
@@ -351,8 +351,8 @@ def run_all_api_calls(in_api_key:str, Matcher:MatcherClass, query_type:str, prog
 
         if save_file:
             print("Saving reference file to: " + api_ref_save_loc[:-5] + ".parquet")
-            Matcher.ref_df.to_parquet(api_ref_save_loc + ".parquet", index=False) # Save checkpoint as well
-            Matcher.ref_df.to_parquet(api_ref_save_loc[:-5] + ".parquet", index=False)
+            Matcher.ref_df.to_parquet(output_folder + api_ref_save_loc + ".parquet", index=False) # Save checkpoint as well
+            Matcher.ref_df.to_parquet(output_folder + api_ref_save_loc[:-5] + ".parquet", index=False)
 
         if Matcher.ref_df.empty:
             print ("No reference data found with API")
@@ -676,8 +676,8 @@ def load_matcher_data(in_text, in_file, in_ref, data_state, results_data_state,
         print("Shape of ref_df after filtering is: ", Matcher.ref_df.shape)
         print("Shape of search_df after filtering is: ", Matcher.search_df.shape)
     
-        Matcher.match_outputs_name = "output/diagnostics_initial_" + today_rev + ".csv" 
-        Matcher.results_orig_df_name = "output/results_initial_" + today_rev + ".csv" 
+        Matcher.match_outputs_name = output_folder + "diagnostics_initial_" + today_rev + ".csv" 
+        Matcher.results_orig_df_name = output_folder + "results_initial_" + today_rev + ".csv" 
     
         Matcher.match_results_output.to_csv(Matcher.match_outputs_name, index = None)
         Matcher.results_on_orig_df.to_csv(Matcher.results_orig_df_name, index = None)
@@ -724,10 +724,6 @@ def run_matcher(in_text:str, in_file:str, in_ref:str, data_state:PandasDataFrame
     InitMatch.ref_df_cleaned = prepare_ref_address(InitMatch.ref_df, InitMatch.ref_address_cols, InitMatch.new_join_col)
     
 
-    # Sort dataframes by postcode - will allow for more efficient matching process if using multiple batches
-    #InitMatch.search_df_cleaned = InitMatch.search_df_cleaned.sort_values(by="postcode")
-    #InitMatch.ref_df_cleaned = InitMatch.ref_df_cleaned.sort_values(by="Postcode")
-
     # Polars implementation - not finalised
     #InitMatch.search_df_cleaned = InitMatch.search_df_cleaned.to_pandas()
     #InitMatch.ref_df_cleaned = InitMatch.ref_df_cleaned.to_pandas()
@@ -777,31 +773,10 @@ def run_matcher(in_text:str, in_file:str, in_ref:str, data_state:PandasDataFrame
 
         search_range = range_df.iloc[row]['search_range']
         ref_range = range_df.iloc[row]['ref_range']
-
-        #print("search_range: ", search_range)
-        #pd.DataFrame(search_range).to_csv("search_range.csv")
-        #print("ref_range: ", ref_range)
         
         BatchMatch = copy.copy(InitMatch)
 
         # Subset the search and reference dfs based on current batch ranges
-        # BatchMatch.search_df = BatchMatch.search_df.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
-        # BatchMatch.search_df_not_matched = BatchMatch.search_df.copy()
-        # BatchMatch.search_df_cleaned = BatchMatch.search_df_cleaned.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
-        # BatchMatch.ref_df = BatchMatch.ref_df.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
-        # BatchMatch.ref_df_cleaned = BatchMatch.ref_df_cleaned.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
-
-
-        # BatchMatch.search_df_after_stand_series = BatchMatch.search_df_after_stand_series.iloc[search_range[0]:search_range[1] + 1]
-        # BatchMatch.ref_df_after_stand_series = BatchMatch.ref_df_after_stand_series.iloc[ref_range[0]:ref_range[1] + 1]
-        # BatchMatch.search_df_after_stand_series_full_stand = BatchMatch.search_df_after_stand_series_full_stand.iloc[search_range[0]:search_range[1] + 1]
-        # BatchMatch.ref_df_after_stand_series_full_stand = BatchMatch.ref_df_after_stand_series_full_stand.iloc[ref_range[0]:ref_range[1] + 1]
-
-        # BatchMatch.search_df_after_stand = BatchMatch.search_df_after_stand.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
-        # BatchMatch.ref_df_after_stand = BatchMatch.ref_df_after_stand.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
-        # BatchMatch.search_df_after_full_stand = BatchMatch.search_df_after_full_stand.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
-        # BatchMatch.ref_df_after_full_stand = BatchMatch.ref_df_after_full_stand.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
-
         BatchMatch.search_df = BatchMatch.search_df[BatchMatch.search_df.index.isin(search_range)].reset_index(drop=True)
         BatchMatch.search_df_not_matched = BatchMatch.search_df.copy()
         BatchMatch.search_df_cleaned = BatchMatch.search_df_cleaned[BatchMatch.search_df_cleaned.index.isin(search_range)].reset_index(drop=True)
@@ -814,25 +789,9 @@ def run_matcher(in_text:str, in_file:str, in_ref:str, data_state:PandasDataFrame
         BatchMatch.search_df_after_full_stand = BatchMatch.search_df_after_full_stand[BatchMatch.search_df_after_full_stand.index.isin(search_range)].reset_index(drop=True)
 
         ### Create lookup lists for fuzzy matches
-        # BatchMatch.search_df_after_stand_series = BatchMatch.search_df_after_stand.copy().set_index('postcode_search')['search_address_stand']
-        # BatchMatch.search_df_after_stand_series_full_stand = BatchMatch.search_df_after_full_stand.copy().set_index('postcode_search')['search_address_stand']
-        # BatchMatch.search_df_after_stand_series = BatchMatch.search_df_after_stand_series.sort_index()
-        # BatchMatch.search_df_after_stand_series_full_stand = BatchMatch.search_df_after_stand_series_full_stand.sort_index()
-
-        #BatchMatch.search_df_after_stand.reset_index(inplace=True, drop = True)
-        #BatchMatch.search_df_after_full_stand.reset_index(inplace=True, drop = True)
-
         BatchMatch.ref_df_after_stand = BatchMatch.ref_df_after_stand[BatchMatch.ref_df_after_stand.index.isin(ref_range)].reset_index(drop=True)
         BatchMatch.ref_df_after_full_stand = BatchMatch.ref_df_after_full_stand[BatchMatch.ref_df_after_full_stand.index.isin(ref_range)].reset_index(drop=True)
 
-        # BatchMatch.ref_df_after_stand_series = BatchMatch.ref_df_after_stand.copy().set_index('postcode_search')['ref_address_stand']
-        # BatchMatch.ref_df_after_stand_series_full_stand = BatchMatch.ref_df_after_full_stand.copy().set_index('postcode_search')['ref_address_stand']
-        # BatchMatch.ref_df_after_stand_series = BatchMatch.ref_df_after_stand_series.sort_index()
-        # BatchMatch.ref_df_after_stand_series_full_stand = BatchMatch.ref_df_after_stand_series_full_stand.sort_index()
-
-        # BatchMatch.ref_df_after_stand.reset_index(inplace=True, drop=True)
-        # BatchMatch.ref_df_after_full_stand.reset_index(inplace=True, drop=True)
-
         # Match the data, unless the search or reference dataframes are empty
         if BatchMatch.search_df.empty or BatchMatch.ref_df.empty:
             out_message = "Nothing to match for batch: " + str(n)
@@ -938,8 +897,6 @@ def create_batch_ranges(df:PandasDataFrame, ref_df:PandasDataFrame, batch_size:i
     df = df.sort_index()
     ref_df = ref_df.sort_index()
 
-    #df.to_csv("batch_search_df.csv")
-
     # Overall batch variables
     batch_indexes = []
     ref_indexes = []
@@ -1184,8 +1141,8 @@ def orchestrate_match_run(Matcher, standardise = False, nnet = False, file_stub=
   
         Matcher.output_summary = create_match_summary(Matcher.match_results_output, df_name = df_name)       
         
-        Matcher.match_outputs_name = "output/diagnostics_" + file_stub + today_rev + ".csv"
-        Matcher.results_orig_df_name = "output/results_" + file_stub + today_rev + ".csv"
+        Matcher.match_outputs_name = output_folder + "diagnostics_" + file_stub + today_rev + ".csv"
+        Matcher.results_orig_df_name = output_folder + "results_" + file_stub + today_rev + ".csv"
     
         Matcher.match_results_output.to_csv(Matcher.match_outputs_name, index = None)
         Matcher.results_on_orig_df.to_csv(Matcher.results_orig_df_name, index = None)
@@ -1233,14 +1190,9 @@ def full_fuzzy_match(search_df:PandasDataFrame,
     # Remove rows from ref search series where postcode is not found in the search_df
     search_df_after_stand_series = search_df_after_stand.copy().set_index('postcode_search')['search_address_stand'].sort_index()
     ref_df_after_stand_series = ref_df_after_stand.copy().set_index('postcode_search')['ref_address_stand'].sort_index()
-
-    #print(search_df_after_stand_series.index.tolist())
-    #print(ref_df_after_stand_series.index.tolist())
-    
+   
     ref_df_after_stand_series_checked = ref_df_after_stand_series.copy()[ref_df_after_stand_series.index.isin(search_df_after_stand_series.index.tolist())]
 
-    # pd.DataFrame(ref_df_after_stand_series_checked.to_csv("ref_df_after_stand_series_checked.csv"))
-
     if len(ref_df_after_stand_series_checked) == 0: 
         print("Nothing relevant in reference data to match!")
         return pd.DataFrame(), pd.DataFrame(),  pd.DataFrame(),pd.DataFrame(),"Nothing relevant in reference data to match!",search_address_cols
@@ -1603,8 +1555,8 @@ def combine_two_matches(OrigMatchClass:MatcherClass, NewMatchClass:MatcherClass,
         ### Rejoin the excluded matches onto the output file
         #NewMatchClass.results_on_orig_df = pd.concat([NewMatchClass.results_on_orig_df, NewMatchClass.excluded_df])
     
-        NewMatchClass.match_outputs_name = "output/diagnostics_" + today_rev + ".csv" # + NewMatchClass.file_name + "_" 
-        NewMatchClass.results_orig_df_name = "output/results_" + today_rev + ".csv" # + NewMatchClass.file_name + "_"
+        NewMatchClass.match_outputs_name = output_folder + "diagnostics_" + today_rev + ".csv" # + NewMatchClass.file_name + "_" 
+        NewMatchClass.results_orig_df_name = output_folder + "results_" + today_rev + ".csv" # + NewMatchClass.file_name + "_"
 
         # Only keep essential columns
         essential_results_cols = [NewMatchClass.search_df_key_field, "Excluded from search", "Matched with reference address", "ref_index", "Reference matched address", "Reference file"]

tools/model_predict.py

@@ -15,10 +15,6 @@ today_rev = datetime.now().strftime("%Y%m%d")
 
 # # Neural net functions
 
-
-
-
-
 def vocab_lookup(characters: str, vocab) -> (int, np.ndarray):
     """
     Taken from the function from the addressnet package by Jason Rigby
@@ -298,21 +294,10 @@ def post_predict_clean(predict_df, orig_search_df, ref_address_cols, search_df_k
 
     predict_df = predict_df.rename(columns={"Postcode":"Postcode_predict"})
 
-    #orig_search_df.to_csv("orig_search_df_pre_predict.csv")
-
     orig_search_df_pc = orig_search_df[[search_df_key_field, "postcode"]].rename(columns={"postcode":"Postcode"}).reset_index(drop=True)
     predict_df = predict_df.merge(orig_search_df_pc, left_index=True, right_index=True, how = "left")
 
-    #predict_df = pd.concat([predict_df, orig_search_df_pc], axis = 1)
-    
-    #predict_df[search_df_key_field] = orig_search_df[search_df_key_field]
-
-    #predict_df = predict_df.drop("index", axis=1)
-
-    #predict_df['index'] = predict_df.index
     predict_df[search_df_key_field] = predict_df[search_df_key_field].astype(str)
-
-    #predict_df.to_csv("predict_end_of_clean.csv")
     
     return predict_df
 

tools/recordlinkage_funcs.py

@@ -93,7 +93,6 @@ def calc_final_nnet_scores(scoresSBM, weights, matching_variables):
         scoresSBM_r = scoresSBM_r.sort_values(by=["level_0","score_perc"], ascending = False)
         
         # Within each search address, remove anything below the max
-        #scoresSBM_r.to_csv("scoresSBM_r.csv")
         scoresSBM_g = scoresSBM_r.reset_index()
         
         # Get maximum score to join on
@@ -114,8 +113,6 @@ def join_on_pred_ref_details(scoresSBM_search_m, ref_search, predict_df_search):
 
     scoresSBM_search_m_j = scoresSBM_search_m_j.reindex(sorted(scoresSBM_search_m_j.columns), axis=1)
 
-    #scoresSBM_search_m_j.to_csv("scoresSBM_search_m_j.csv") 
-
     return scoresSBM_search_m_j 
 
 def rearrange_columns(scoresSBM_search_m_j, new_join_col, search_df_key_field, blocker_column, standardise):
@@ -175,14 +172,10 @@ def rearrange_columns(scoresSBM_search_m_j, new_join_col, search_df_key_field, b
     
     scoresSBM_out = scoresSBM_search_m_j[final_cols]
 
-    #scoresSBM_out.to_csv("scoresSBM_out" + "_" + blocker_column[0] + "_" + str(standardise) + ".csv")
-
     return scoresSBM_out, start_columns
 
 def create_matched_results_nnet(scoresSBM_best, search_df_key_field, orig_search_df, new_join_col, standardise, ref_search, blocker_column, score_cut_off):
 
-    #scoresSBM_best.to_csv("scores_sbm_best_" + str(standardise) + ".csv")
-
     ### Make the final 'matched output' file
     scoresSBM_best_pred_cols = scoresSBM_best.filter(regex='_pred$').iloc[:,1:-1]
     scoresSBM_best["search_orig_address"] = (scoresSBM_best_pred_cols.agg(' '.join, axis=1)).str.strip().str.replace("\s{2,}", " ", regex=True)
@@ -199,22 +192,16 @@ def create_matched_results_nnet(scoresSBM_best, search_df_key_field, orig_search
         'full_match_score_based', 'Reference file']], on = search_df_key_field, how = "left").\
         rename(columns={"full_address":"search_orig_address"})
     
-    #ref_search.to_csv("ref_search.csv")
-
     if 'index' not in ref_search.columns:
         ref_search['ref_index'] = ref_search.index
 
     matched_output_SBM = matched_output_SBM.merge(ref_search.drop_duplicates("fulladdress")[["ref_index", "fulladdress", "Postcode", "property_number", "prop_number", "flat_number", "apart_number", "block_number", 'unit_number', "room_number", "house_court_name", "ref_address_stand"]], left_on = "address_ref", right_on = "fulladdress", how = "left", suffixes=('_search', '_reference')).rename(columns={"fulladdress":"reference_orig_address", "ref_address_stand":"reference_list_address"})
 
-    #matched_output_SBM.to_csv("matched_output_SBM_earlier_" + str(standardise) + ".csv")
-
     # To replace with number check
    
-
     matched_output_SBM = matched_output_SBM.rename(columns={"full_match_score_based":"full_match"})
 
     matched_output_SBM['property_number_match'] = matched_output_SBM['full_match']
-    #
 
     scores_SBM_best_cols = [search_df_key_field, 'full_match_score_based',  'perc_weighted_columns_matched', 'address_pred']#, "reference_mod_address"]
     scores_SBM_best_cols.extend(new_join_col)
@@ -223,20 +210,13 @@ def create_matched_results_nnet(scoresSBM_best, search_df_key_field, orig_search
 
     matched_output_SBM = matched_output_SBM.merge(matched_output_SBM_b.drop_duplicates(search_df_key_field), on = search_df_key_field,  how = "left")
 
-    #matched_output_SBM.to_csv("matched_output_SBM_later_" + str(standardise) + ".csv")
-
     from tools.fuzzy_match import create_diag_shortlist
     matched_output_SBM = create_diag_shortlist(matched_output_SBM, "search_orig_address", score_cut_off, blocker_column, fuzzy_col='perc_weighted_columns_matched', search_mod_address="address_pred", resolve_tie_breaks=False)
 
-    #matched_output_SBM.to_csv("matched_output_after.csv")
-    
-    #matched_output_SBM["UPRN"] = scoresSBM_best['UPRN']
 
     matched_output_SBM['standardised_address'] = standardise
 
-    matched_output_SBM = matched_output_SBM.rename(columns={"address_pred":"search_mod_address",
-        #"address_ref":"reference_orig_address",
-        #"full_match_score_based":"fuzzy_score_match",                        
+    matched_output_SBM = matched_output_SBM.rename(columns={"address_pred":"search_mod_address",                      
         'perc_weighted_columns_matched':"fuzzy_score"})
     
     matched_output_SBM_cols = [search_df_key_field, 'search_orig_address','reference_orig_address',
@@ -257,10 +237,6 @@ def create_matched_results_nnet(scoresSBM_best, search_df_key_field, orig_search
         "unit_number_search","unit_number_reference",
         'house_court_name_search', 'house_court_name_reference',
         "search_mod_address", 'reference_mod_address','Postcode', 'postcode', 'ref_index', 'Reference file']
-
-    #matched_output_SBM_cols = [search_df_key_field, 'search_orig_address', 'reference_orig_address',
-    #'full_match', 'fuzzy_score_match', 'property_number_match', 'full_number_match', 
-    #'fuzzy_score', 'search_mod_address', 'reference_mod_address', 'Reference file']
     
     matched_output_SBM_cols.extend(new_join_col)
     matched_output_SBM_cols.extend(['standardised_address'])
@@ -268,8 +244,6 @@ def create_matched_results_nnet(scoresSBM_best, search_df_key_field, orig_search
     
     matched_output_SBM = matched_output_SBM.sort_values(search_df_key_field, ascending=True)
 
-    #matched_output_SBM.to_csv("matched_output_SBM_out.csv")
-
     return matched_output_SBM
 
 def score_based_match(predict_df_search, ref_search, orig_search_df, matching_variables, text_columns, blocker_column,  weights, fuzzy_method, score_cut_off, search_df_key_field, standardise, new_join_col, score_cut_off_nnet_street=score_cut_off_nnet_street):
@@ -287,8 +261,6 @@ def score_based_match(predict_df_search, ref_search, orig_search_df, matching_va
 
     scoresSBM_search_m_j = join_on_pred_ref_details(scoresSBM_search_m, ref_search, predict_df_search)
 
-    #scoresSBM_search_m_j.to_csv("scoresSBM_search_m_j.csv")
-
     # When blocking by street, may to have an increased threshold as this is more prone to making mistakes
     if blocker_column[0] == "Street": scoresSBM_search_m_j['full_match_score_based'] = (scoresSBM_search_m_j['score_perc'] >= score_cut_off_nnet_street)
 
@@ -297,15 +269,10 @@ def score_based_match(predict_df_search, ref_search, orig_search_df, matching_va
     ### Reorder some columns
     scoresSBM_out, start_columns = rearrange_columns(scoresSBM_search_m_j, new_join_col, search_df_key_field, blocker_column, standardise)
 
-    #scoresSBM_out.to_csv("scoresSBM_out.csv")
-
     matched_output_SBM = create_matched_results_nnet(scoresSBM_out, search_df_key_field, orig_search_df, new_join_col, standardise, ref_search, blocker_column, score_cut_off)
 
     matched_output_SBM_best = matched_output_SBM.sort_values([search_df_key_field, "full_match"], ascending = [True, False]).drop_duplicates(search_df_key_field)
 
-    #matched_output_SBM.to_csv("matched_output_SBM.csv")
-    #matched_output_SBM_best.to_csv("matched_output_SBM_best.csv")
-
     scoresSBM_best = scoresSBM_out[scoresSBM_out[search_df_key_field].isin(matched_output_SBM_best[search_df_key_field])]
 
     return scoresSBM_best, matched_output_SBM_best