updating new preprocess

app.py

@@ -20,12 +20,29 @@ def zero_shot(doc, candidates):
     return dict(zip(labels, scores))
 
 #define a function to preprocess transaction query
+# def preprocess(transaction):
+#     pattern = r'([A-Za-z0-9\s]+)(?:/| |$)'
+#     match = re.search(pattern, transaction)
+#     if match:
+#         return match.group(1).strip()
+#     return None
+
 def preprocess(transaction):
-    pattern = r'([A-Za-z0-9\s]+)(?:/| |$)'
-    match = re.search(pattern, transaction)
-    if match:
-        return match.group(1).strip()
-    return None
+    remove_words = ["pos", "mps", "bil", "onl"]
+
+    # Convert to lowercase
+    transaction = transaction.lower()
+
+    # Remove unwanted words
+    for word in remove_words:
+        transaction = transaction.replace(word, "")
+
+    # Remove special characters and digits
+    transaction = re.sub(r"[^a-z\s]+", "", transaction)
+
+    # Remove extra spaces
+    transaction = re.sub(r"\s+", " ", transaction).strip()
+    return transaction
 
 
 #create input and output objects