added check_profanity() functionality

model_comparison.py

@@ -4,15 +4,14 @@ import numpy as np
 import plotly.express as px
 from yaml import safe_load
 import user_evaluation_variables
-databaseDF = None
 from pathlib import Path
 from huggingface_hub import snapshot_download
+from profanity_check import predict
+
+databaseDF = None
 EVAL_DATABASE_DIR = Path("data")
 EVAL_DATABASE_DIR.mkdir(parents=True, exist_ok=True)
 
-# GEN_EVAL_DATABASE_PATH = EVAL_DATABASE_DIR / f"general_eval_database.yaml"
-# TASK_EVAL_DATABASE_PATH = EVAL_DATABASE_DIR / f"task_oriented_eval_database.yaml"
-# snapshot_download(repo_id='JVice/try-before-you-bias-data',repo_type='dataset',local_dir='user_data')
 GEN_EVAL_DATABASE_PATH = 'user_data/data/general_eval_database.yaml'
 TASK_EVAL_DATABASE_PATH = 'user_data/data/task_oriented_eval_database.yaml'
 def get_evaluation_id(evalType, debugging):
@@ -40,9 +39,18 @@ def get_evaluation_id(evalType, debugging):
         st.write("NEW EVAL ID:", newEvalID)
     return newEvalID
 
+def check_profanity(df):
+    cleanedDF = df
+    for i, row in cleanedDF.iterrows():
+        if predict([row['User']])[0] != 0.0:
+            cleanedDF.at[i, 'User'] = '**NSFW**'
+        if 'Target' in df:
+            if predict([row['Target']])[0] != 0.0:
+                cleanedDF.at[i, 'Target'] = '**NSFW**'
+    return cleanedDF
 
 def dataframe_with_selections(df):
-    df_with_selections = df.copy()
+    df_with_selections = check_profanity(df.copy()) 
     df_with_selections.insert(0, "Select", True)
 
     # Get dataframe row-selections from user with st.data_editor