Update app.py

app.py

@@ -215,17 +215,9 @@ def custom_tokenize(text, tokenizer):
     return tokenizer.encode(text).tokens
 
 # Embedding and Vector Store
-@lru_cache(maxsize=None)
-def get_embedding_model(model_type, model_name):
-    model_path = model_manager.get_model(model_type, model_name)
-    if model_type == 'HuggingFace':
-        return HuggingFaceEmbeddings(model_name=model_path)
-    elif model_type == 'OpenAI':
-        return OpenAIEmbeddings(model=model_path)
-    elif model_type == 'Cohere':
-        return CohereEmbeddings(model=model_path)
-    else:
-        raise ValueError(f"Unsupported model type: {model_type}")
+#@lru_cache(maxsize=None)
+
+# Helper functions
 
 def get_text_splitter(split_strategy, chunk_size, overlap_size, custom_separators=None):
     if split_strategy == 'token':
@@ -239,9 +231,35 @@ def get_text_splitter(split_strategy, chunk_size, overlap_size, custom_separator
     else:
         raise ValueError(f"Unsupported split strategy: {split_strategy}")
 
+def get_embedding_model(model_type, model_name):
+    model_path = model_manager.get_model(model_type, model_name)
+    if model_type == 'HuggingFace':
+        return HuggingFaceEmbeddings(model_name=model_path)
+    elif model_type == 'OpenAI':
+        return OpenAIEmbeddings(model=model_path)
+    elif model_type == 'Cohere':
+        return CohereEmbeddings(model=model_path)
+    else:
+        raise ValueError(f"Unsupported model type: {model_type}")
+
 def get_vector_store(vector_store_type, chunks, embedding_model):
     chunks_tuple = tuple(chunks)
-    return _create_vector_store(vector_store_type, chunks_tuple, embedding_model)
+    if vector_store_type == 'FAISS':
+        return FAISS.from_texts(chunks, embedding_model)
+    elif vector_store_type == 'Chroma':
+        return Chroma.from_texts(chunks, embedding_model)
+    else:
+        raise ValueError(f"Unsupported vector store type: {vector_store_type}")
+
+def get_retriever(vector_store, search_type, search_kwargs):
+    if search_type == 'similarity':
+        return vector_store.as_retriever(search_type="similarity", search_kwargs=search_kwargs)
+    elif search_type == 'mmr':
+        return vector_store.as_retriever(search_type="mmr", search_kwargs=search_kwargs)
+    elif search_type == 'custom':
+        return vector_store.as_retriever(search_type="similarity", search_kwargs=search_kwargs)
+    else:
+        raise ValueError(f"Unsupported search type: {search_type}")
 
 def custom_similarity(query_embedding, doc_embedding, query, doc_text, phonetic_weight=0.3):
     embedding_sim = np.dot(query_embedding, doc_embedding) / (np.linalg.norm(query_embedding) * np.linalg.norm(doc_embedding))
@@ -259,18 +277,9 @@ def _create_vector_store(vector_store_type, chunks_tuple, embedding_model):
     else:
         raise ValueError(f"Unsupported vector store type: {vector_store_type}")
 
-def get_retriever(vector_store, search_type, search_kwargs):
-    if search_type == 'similarity':
-        return vector_store.as_retriever(search_type="similarity", search_kwargs=search_kwargs)
-    elif search_type == 'mmr':
-        return vector_store.as_retriever(search_type="mmr", search_kwargs=search_kwargs)
-    elif search_type == 'custom':
-        return vector_store.as_retriever(search_type="similarity", search_kwargs=search_kwargs)
-    else:
-        raise ValueError(f"Unsupported search type: {search_type}")
 
 # Main Processing Functions
-def process_files(file_path, model_type, model_name, split_strategy, chunk_size, overlap_size, custom_separators, lang='german', custom_tokenizer_file=None, custom_tokenizer_model=None, custom_tokenizer_vocab_size=10000, custom_tokenizer_special_tokens=None):
+def process_files(file_path, model_type, model_name, split_strategy, chunk_size, overlap_size, custom_separators, lang='german', apply_preprocessing=True, custom_tokenizer_file=None, custom_tokenizer_model=None, custom_tokenizer_vocab_size=10000, custom_tokenizer_special_tokens=None):
     if file_path:
         text = FileHandler.extract_text(file_path)
     else:
@@ -282,7 +291,7 @@ def process_files(file_path, model_type, model_name, split_strategy, chunk_size,
     if custom_tokenizer_file:
         tokenizer = create_custom_tokenizer(custom_tokenizer_file, custom_tokenizer_model, custom_tokenizer_vocab_size, custom_tokenizer_special_tokens)
         text = ' '.join(custom_tokenize(text, tokenizer))
-    else:
+    elif apply_preprocessing:
         text = preprocess_text(text, lang)
 
     text_splitter = get_text_splitter(split_strategy, chunk_size, overlap_size, custom_separators)
@@ -292,8 +301,8 @@ def process_files(file_path, model_type, model_name, split_strategy, chunk_size,
 
     return chunks, embedding_model, len(text.split())
 
-def search_embeddings(chunks, embedding_model, vector_store_type, search_type, query, top_k, lang='german', phonetic_weight=0.3):
-    preprocessed_query = preprocess_text(query, lang)
+def search_embeddings(chunks, embedding_model, vector_store_type, search_type, query, top_k, lang='german', apply_phonetic=True, phonetic_weight=0.3):
+    preprocessed_query = preprocess_text(query, lang) if apply_phonetic else query
     
     vector_store = get_vector_store(vector_store_type, chunks, embedding_model)
     retriever = get_retriever(vector_store, search_type, {"k": top_k})
@@ -303,8 +312,11 @@ def search_embeddings(chunks, embedding_model, vector_store_type, search_type, q
 
     def score_result(doc):
         similarity_score = vector_store.similarity_search_with_score(doc.page_content, k=1)[0][1]
-        phonetic_score = phonetic_match(doc.page_content, query)
-        return (1 - phonetic_weight) * similarity_score + phonetic_weight * phonetic_score
+        if apply_phonetic:
+            phonetic_score = phonetic_match(doc.page_content, query)
+            return (1 - phonetic_weight) * similarity_score + phonetic_weight * phonetic_score
+        else:
+            return similarity_score
 
     results = sorted(results, key=score_result, reverse=True)
     end_time = time.time()
@@ -323,6 +335,8 @@ def search_embeddings(chunks, embedding_model, vector_store_type, search_type, q
 
     return results_df, end_time - start_time, vector_store, results
 
+
+
 # Evaluation Metrics
 # ... (previous code remains the same)