Upgraded to Gradio 4.16.0. Added Spacy fuzzy search functionality.

README.md

@@ -4,7 +4,7 @@ emoji: 🔍
 colorFrom: purple
 colorTo: green
 sdk: gradio
-sdk_version: 3.50.2
+sdk_version: 4.16.0
 app_file: app.py
 pinned: false
 license: apache-2.0

app.py

@@ -3,6 +3,8 @@ from search_funcs.bm25_functions import prepare_bm25_input_data, prepare_bm25, b
 #from search_funcs.semantic_ingest_functions import parse_csv_or_excel, csv_excel_text_to_docs
 #from search_funcs.semantic_functions import docs_to_jina_embed_np_array, jina_simple_retrieval
 from search_funcs.helper_functions import dummy_function, display_info, initial_data_load, put_columns_in_join_df, get_temp_folder_path, empty_folder
+from search_funcs.spacy_search_funcs import spacy_fuzzy_search
+
 
 import gradio as gr
 import pandas as pd
@@ -33,6 +35,7 @@ with block:
 
     corpus_state = gr.State()
     keyword_data_state = gr.State(pd.DataFrame())
+    keyword_data_list_state = gr.State([])
     join_data_state = gr.State(pd.DataFrame())
     semantic_data_state = gr.State(pd.DataFrame())
 
@@ -74,15 +77,15 @@ depends on factors such as the type of documents or queries. Information taken f
                 load_finished_message = gr.Textbox(label="Load progress", scale = 2)
 
         with gr.Accordion(label = "Search data", open=True):
+            keyword_query = gr.Textbox(label="Enter your search term")
             with gr.Row():
-                keyword_query = gr.Textbox(label="Enter your search term")
-                             
-            keyword_search_button = gr.Button(value="Search text")
-
+                keyword_search_button = gr.Button(value="Keyword search", variant="primary")
+                fuzzy_search_button = gr.Button(value="Fuzzy search (much slower)", variant="secondary")
             with gr.Row():
                 output_single_text = gr.Textbox(label="Top result")
                 output_file = gr.File(label="File output")
 
+
     # with gr.Tab("Semantic search"):
     #     gr.Markdown(
     # """
@@ -131,8 +134,10 @@ depends on factors such as the type of documents or queries. Information taken f
                 in_no_search_results_button = gr.Button(value = "Search results number info", scale = 1)
             with gr.Row():
                 in_search_param_button = gr.Button(value="Load search parameters (Need to click this if you changed anything above)")
-        with gr.Accordion(label="Semantic search options", open = False):
-            semantic_min_distance = gr.Slider(label = "Minimum distance score for search result to be included", value = 0.75, minimum=0, maximum=0.95, step=0.01)
+        with gr.Accordion(label="Fuzzy search options", open = False):
+                no_spelling_mistakes = gr.Slider(label = "Number of spelling mistakes allowed in fuzzy search", value = 1, minimum=1, maximum=4, step=1)
+       # with gr.Accordion(label="Semantic search options", open = False):
+       #     semantic_min_distance = gr.Slider(label = "Minimum distance score for search result to be included", value = 0.75, minimum=0, maximum=0.95, step=0.01)
         with gr.Accordion(label = "Join on additional dataframes to results", open = False):
             in_join_file = gr.File(label="Upload your data to join here")
             in_join_message = gr.Textbox(label="Join file load progress")
@@ -153,12 +158,16 @@ depends on factors such as the type of documents or queries. Information taken f
     in_join_file.upload(put_columns_in_join_df, inputs=[in_join_file], outputs=[in_join_column, join_data_state, in_join_message])
  
     # Load in BM25 data
-    load_bm25_data_button.click(fn=prepare_bm25_input_data, inputs=[in_bm25_file, in_bm25_column, keyword_data_state, tokenised_state, in_clean_data, return_intermediate_files], outputs=[corpus_state, load_finished_message, keyword_data_state, output_file, output_file]).\
+    load_bm25_data_button.click(fn=prepare_bm25_input_data, inputs=[in_bm25_file, in_bm25_column, keyword_data_state, tokenised_state, in_clean_data, return_intermediate_files], outputs=[corpus_state, load_finished_message, keyword_data_state, output_file, output_file, keyword_data_list_state]).\
     then(fn=prepare_bm25, inputs=[corpus_state, in_bm25_file, in_bm25_column, search_index_state, in_clean_data, return_intermediate_files, in_k1, in_b, in_alpha], outputs=[load_finished_message, output_file])#.\
     
     # BM25 search functions on click or enter
     keyword_search_button.click(fn=bm25_search, inputs=[keyword_query, in_no_search_results, keyword_data_state, in_bm25_column, join_data_state, in_clean_data, in_join_column, search_df_join_column], outputs=[output_single_text, output_file], api_name="keyword")
     keyword_query.submit(fn=bm25_search, inputs=[keyword_query, in_no_search_results, keyword_data_state, in_bm25_column, join_data_state, in_clean_data, in_join_column, search_df_join_column], outputs=[output_single_text, output_file])
+
+    # Fuzzy search functions on click
+
+    fuzzy_search_button.click(fn=spacy_fuzzy_search, inputs=[keyword_query, keyword_data_list_state, keyword_data_state, in_bm25_column, join_data_state, search_df_join_column, in_join_column, no_spelling_mistakes], outputs=[output_single_text, output_file], api_name="fuzzy")
     
     ### SEMANTIC SEARCH ###
     # Load in a csv/excel file for semantic search

requirements.txt

@@ -7,4 +7,4 @@ openpyxl==3.1.2
 # torch==2.1.2
 spacy==3.7.2
 en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1.tar.gz
-gradio==3.50.2
+gradio==4.16.0

search_funcs/bm25_functions.py

@@ -236,7 +236,7 @@ def prepare_bm25_input_data(in_file, text_column, data_state, tokenised_state, c
 
 	if not in_file:
 		print("No input file found. Please load in at least one file.")
-		return None, "No input file found. Please load in at least one file.", data_state, None, None, None
+		return None, "No input file found. Please load in at least one file.", data_state, None, None, None, []
 
 	progress(0, desc = "Loading in data")
 	file_list = [string.name for string in in_file]
@@ -246,10 +246,10 @@ def prepare_bm25_input_data(in_file, text_column, data_state, tokenised_state, c
 	data_file_names = [string for string in file_list if "tokenised" not in string.lower() and "npz" not in string.lower() and "gz" not in string.lower()]
 
 	if not data_file_names:
-		return None, "Please load in at least one csv/Excel/parquet data file.", data_state, None, None, None
+		return None, "Please load in at least one csv/Excel/parquet data file.", data_state, None, None, None, []
 
 	if not text_column:
-		return None, "Please enter a column name to search.", data_state, None, None, None
+		return None, "Please enter a column name to search.", data_state, None, None, None, []
 
 	data_file_name = data_file_names[0]
 
@@ -329,9 +329,9 @@ def prepare_bm25_input_data(in_file, text_column, data_state, tokenised_state, c
 
 		pd.DataFrame(data={"Corpus":corpus}).to_parquet(tokenised_data_file_name)
 
-		return corpus, message, df, out_file_name, tokenised_data_file_name
+		return corpus, message, df, out_file_name, tokenised_data_file_name, df_list
 
-	return corpus, message, df, out_file_name, None # tokenised_data_file_name
+	return corpus, message, df, out_file_name, None, df_list
 
 def save_prepared_bm25_data(in_file_name, prepared_text_list, in_df, in_bm25_column, progress=gr.Progress(track_tqdm=True)):
 
@@ -506,7 +506,7 @@ def bm25_search(free_text_query, in_no_search_results, original_data, text_colum
 		# Duplicates dropped so as not to expand out dataframe
 		join_df = join_df.drop_duplicates(in_join_column)
 
-		results_df_out = results_df_out.merge(join_df,left_on=search_df_join_column, right_on=in_join_column, how="left")#.drop(in_join_column, axis=1)
+		results_df_out = results_df_out.merge(join_df,left_on=search_df_join_column, right_on=in_join_column, how="left", suffixes=('','_y'))#.drop(in_join_column, axis=1)
 
 	# Reorder results by score
 	results_df_out = results_df_out.sort_values('search_score_abs', ascending=False)

search_funcs/helper_functions.py

@@ -72,11 +72,11 @@ def read_file(filename):
     print("Loading in file")
 
     if file_type == 'csv':
-        file = pd.read_csv(filename, low_memory=False).reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
+        file = pd.read_csv(filename, low_memory=False).reset_index()#.drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
     elif file_type == 'xlsx':
-        file = pd.read_excel(filename).reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
+        file = pd.read_excel(filename).reset_index()#.drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
     elif file_type == 'parquet':
-        file = pd.read_parquet(filename).reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
+        file = pd.read_parquet(filename).reset_index()#.drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
     elif file_type == 'pkl.gz':
         with gzip.open(filename, 'rb') as file:
             file = pickle.load(file)

search_funcs/semantic_functions.py

@@ -214,7 +214,7 @@ def process_data_from_scores_df(df_docs, in_join_file, out_passages, vec_score_c
 
         results_df_out[search_df_join_column] = results_df_out[search_df_join_column].astype(str).str.replace("\.0$","", regex=True)
 
-        results_df_out = results_df_out.merge(join_df,left_on=search_df_join_column, right_on=in_join_column, how="left")#.drop(in_join_column, axis=1)
+        results_df_out = results_df_out.merge(join_df,left_on=search_df_join_column, right_on=in_join_column, how="left", suffixes=('','_y'))#.drop(in_join_column, axis=1)
 
     return results_df_out
 

search_funcs/spacy_search_funcs.py

@@ -4,16 +4,19 @@ import numpy as np
 import gradio as gr
 import pandas as pd
 from typing import List, Type
+from datetime import datetime
 
 PandasDataFrame = Type[pd.DataFrame]
 
+today_rev = datetime.now().strftime("%Y%m%d")
+
 nlp = spacy.load("en_core_web_sm")
 
 string_query = "knife attack run fast"
 df_list = ["Last week someone was grievously injured in a knife attack on Exmoor road. Running away. They ran as fast as possible. I run.","This is the 3rd knifing in the area in as many weeks; knives everywhere.", "attacks of this kind have been increasing for years. Knife attack or knife attack.", "Nothing happened here"]
 
 
-def spacy_fuzzy_search(string_query:str, df_list: List[str], original_data: PandasDataFrame, search_df_join_column:str, in_join_column:str, no_spelling_mistakes:int = 1, progress=gr.Progress(track_tqdm=True)):
+def spacy_fuzzy_search(string_query:str, df_list: List[str], original_data: PandasDataFrame, text_column:str, in_join_file: PandasDataFrame, search_df_join_column:str, in_join_column:str, no_spelling_mistakes:int = 1, progress=gr.Progress(track_tqdm=True)):
     ''' Conduct fuzzy match on a list of data.'''
 
     query = nlp(string_query)
@@ -26,52 +29,17 @@ def spacy_fuzzy_search(string_query:str, df_list: List[str], original_data: Pand
     if len(tokenised_query) > 1:
         pattern_lemma = [{"LEMMA": {"IN": tokenised_query}}]
         pattern_fuzz = [{"TEXT": {spelling_mistakes_fuzzy_pattern: {"IN": tokenised_query}}}]
-    elif len(tokenised_query) == 1:
+    else:
         pattern_lemma = [{"LEMMA": tokenised_query[0]}]
         pattern_fuzz = [{"TEXT": {spelling_mistakes_fuzzy_pattern: tokenised_query[0]}}]
-    else:
-        tokenised_query = [""]
-
-    # %%
-    search_pattern = pattern_fuzz.copy()
-    search_pattern.extend(pattern_lemma)
 
-  
+   
     # %%
     matcher = Matcher(nlp.vocab)
-
-    # %% [markdown]
-    # from spacy.tokens import Span
-    # from spacy import displacy
-    # 
-    # def add_event_ent(matcher, doc, i, matches):
-    #     # Get the current match and create tuple of entity label, start and end.
-    #     # Append entity to the doc's entity. (Don't overwrite doc.ents!)
-    #     match_id, start, end = matches[i]
-    #     entity = Span(doc, start, end, label="EVENT")
-    #     doc.ents += (entity,)
-    #     print(entity.text)
-
-    # %% [markdown]
-    # matched_sents = []  # Collect data of matched sentences to be visualized
-    # 
-    # def collect_sents(matcher, doc, i, matches):
-    #     match_id, start, end = matches[i]
-    #     span = doc[start:end]  # Matched span
-    #     sent = span.sent  # Sentence containing matched span
-    #     # Append mock entity for match in displaCy style to matched_sents
-    #     # get the match span by ofsetting the start and end of the span with the
-    #     # start and end of the sentence in the doc
-    #     match_ents = [{
-    #         "start": span.start_char - sent.start_char,
-    #         "end": span.end_char - sent.start_char,
-    #         "label": "MATCH",
-    #     }]
-    #     matched_sents.append({"text": sent.text, "ents": match_ents})
-
+    
     # %%
-    matcher.add(string_query, [pattern_fuzz])#, on_match=add_event_ent)
-    matcher.add(string_query, [pattern_lemma])#, on_match=add_event_ent)
+    matcher.add(string_query, [pattern_fuzz])
+    matcher.add(string_query, [pattern_lemma])
 
     # %%
     batch_size = 256
@@ -100,8 +68,11 @@ def spacy_fuzzy_search(string_query:str, df_list: List[str], original_data: Pand
     results_df = pd.DataFrame(data={"index": list(range(len(df_list))),
                                     "search_text": df_list,
                                     "search_score_abs": match_scores})
-    results_df['search_score_abs'] = abs(round(results_df['search_score_abs'], 2))
-    results_df_out = results_df[['index', 'search_text', 'search_score_abs']].merge(original_data,left_on="index", right_index=True, how="left")#.drop("index", axis=1)
+    results_df['search_score_abs'] = abs(round(results_df['search_score_abs']*100, 2))
+    results_df_out = results_df[['index', 'search_text', 'search_score_abs']].merge(original_data,left_on="index", right_index=True, how="left")
+
+    # Keep only results with at least one match
+    results_df_out = results_df_out.loc[results_df["search_score_abs"] > 0, :]
 
     # Join on additional files
     if not in_join_file.empty:
@@ -113,13 +84,13 @@ def spacy_fuzzy_search(string_query:str, df_list: List[str], original_data: Pand
         # Duplicates dropped so as not to expand out dataframe
         join_df = join_df.drop_duplicates(in_join_column)
 
-        results_df_out = results_df_out.merge(join_df,left_on=search_df_join_column, right_on=in_join_column, how="left")#.drop(in_join_column, axis=1)
+        results_df_out = results_df_out.merge(join_df,left_on=search_df_join_column, right_on=in_join_column, how="left", suffixes=('','_y'))#.drop(in_join_column, axis=1)
 
     # Reorder results by score
     results_df_out = results_df_out.sort_values('search_score_abs', ascending=False)
 
     # Out file
-    query_str_file = ("_").join(token_query)
+    query_str_file = ("_").join(tokenised_query)
     results_df_name = "keyword_search_result_" + today_rev + "_" +  query_str_file + ".xlsx"
 
     print("Saving search file output")
@@ -130,8 +101,4 @@ def spacy_fuzzy_search(string_query:str, df_list: List[str], original_data: Pand
 
     print("Returning results")
 
-    return results_first_text, results_df_name
-
-
-match_list = spacy_fuzzy_search(string_query, df_list)
-print(match_list)
+    return results_first_text, results_df_name