update scripts

src/backend/model_operations.py

@@ -35,7 +35,7 @@ import spacy_transformers
 import subprocess
 
 # Run the command to download the spaCy model
-subprocess.run(["python", "-m", "spacy", "download", "en_core_web_lg"], check=True)
+# subprocess.run(["python", "-m", "spacy", "download", "en_core_web_lg"], check=True)
 # subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"], check=True)
 # subprocess.run(["pip", "install", "spacy-transformers"], check=True)
 # subprocess.run(["pip", "install", "curated-transformers"], check=True)
@@ -43,7 +43,7 @@ subprocess.run(["python", "-m", "spacy", "download", "en_core_web_lg"], check=Tr
 # Load spacy model for word tokenization
 # nlp = spacy.load("en_core_web_sm")
 try:
-    nlp1 = spacy.load("en_core_web_lg")
+    nlp1 = spacy.load("en_core_web_sm")
 except OSError:
     print("Can not load spacy model")
 
@@ -171,7 +171,8 @@ class ResponseGenerator:
                     # print(ID, q_ID, prompt_value)
                     system_prompt = envs.SYSTEM_PROMPT
                     _user_prompt = prompt_value
-                    for ii in range(50):
+                    print(_user_prompt)
+                    for ii in range(100):
                     # user_prompt = f"{envs.USER_PROMPT}\nPassage:\n{_source}"
                         while True:
                             try:
@@ -179,6 +180,7 @@ class ResponseGenerator:
                                 print(self.model_id.lower(),'-',ID,'-',j,'-',ii)
 
                                 _response = self.send_request(system_prompt, _user_prompt)
+                                # print(_response)
                                 # print(f"Finish index {index}")
                                 break
                             except Exception as e:
@@ -205,6 +207,7 @@ class ResponseGenerator:
                                         time.sleep(wait_time)
                                         try:
                                             _response = self.send_request(system_prompt, _user_prompt)
+
                                             break
                                         except Exception as ee:
                                             exceptions.append(ee)
@@ -512,7 +515,7 @@ class EvaluationModel:
         self.scores = []
         self.humanlike_score = None
 
-    def code_results_llm(self, responses_df):
+    def code_results_llm_cleaned(self, responses_df):
         '''code results from LLM's response'''
         output = []
         '''database for Exp4'''
@@ -738,25 +741,365 @@ class EvaluationModel:
                 doc = nlp1(sentence)
                 subject = "None"
                 obj = "None"
+                pobj_list = []  # To collect all prepositional objects
+
+                for token in doc:
+                    if token.dep_ == "nsubj":
+                        subject = token.text
+                    elif token.dep_ == "dobj":
+                        obj = token.text
+                    elif token.dep_ == "pobj":
+                        pobj_list.append(token.text)  # Collect prepositional objects
+
+                rs_list = rs.lower().split()
+                if subject in rs_list and (obj in rs_list or any(pobj == r for pobj in pobj_list for r in rs_list)):
+                    output.append("Other")
+                elif subject in rs_list:
+                    output.append("VP")
+                elif obj in rs_list or any(pobj == r for pobj in pobj_list for r in rs_list):
+                    output.append("NP")
+                else:
+                    output.append("Other")
+
+                '''Exp7'''
+            elif responses_df["Experiment"][i] == "E7":
+                # rs = responses_df["Response"][i].strip().lower()
+                rs = rs.replace(".", "").replace(",", "").lower()
+                #print("E7", rs)
+                if "yes" in rs and "no" in rs:
+                    output.append("Other")
+                elif "no" in rs:
+                    output.append("0")
+                elif "yes" in rs:
+                    output.append("1")
+                else:
+                    output.append("Other")
+
+                '''Exp8'''
+            elif responses_df["Experiment"][i] == "E8":
+                # rs = responses_df["Response"][i].strip()
+                #print("E8", rs)
+                if "something is wrong with the question" in rs:
+                    output.append("1")
+                else:
+                    output.append("0")
+
+                '''Exp9'''
+            elif responses_df["Experiment"][i] == "E9":
+                male, female = 0, 0
+
+                # rs = responses_df["Response"][i].strip()
+                if "because" in rs:
+                    rs = rs.replace("because because", "because").split("because")[1]
+                else:
+                    rs = rs
+                condition = responses_df["Factor 2"][i].strip()
+                rs = rs.split(" ")
+                for w in rs:
+                    if w in male_keyword and female != 1:
+                        male = 1
+                        break
+                    if w in female_keyword and male != 1:
+                        female = 1
+                        break
+                #print("E9", "condition", condition, "male", male, "female", female)
+                if male == 0 and female == 0:
+                    output.append('Other')
+                else:
+                    if male == 1 and female == 0:
+                        if condition == "MF":
+                            output.append("Subject")
+                        elif condition == "FM":
+                            output.append("Object")
+                        else:
+                            output.append("Other")
+                    elif female == 1 and male == 0:
+                        if condition == "MF":
+                            output.append("Object")
+                        elif condition == "FM":
+                            output.append("Subject")
+                        else:
+                            output.append("Other")
+
+                '''Exp10'''
+            elif responses_df["Experiment"][i] == "E10":
+                # rs = responses_df["Response"][i].strip()
+                rs = rs.replace(".", "")
+                if rs == "yes":
+                    output.append("1")
+                else:
+                    output.append("0")
+            else:
+                #print("can;t find the Exp:", responses_df["Experiment"][i])
+                output.append("NA")
+            # print(output)
+        # exit()
+                '''LLM'''
+        print(len(output))
+        import re
+        def clean_text(text):
+            if isinstance(text, str):
+                return re.sub(r'[^\x00-\x7F]+', '', text)
+            return text
+
+        responses_df["Experiment"] = responses_df["Experiment"].apply(clean_text)
+        responses_df["Question_ID"] = responses_df["Question_ID"].apply(clean_text)
+        responses_df["Item"] = responses_df["Item"].apply(clean_text)
+        responses_df["Response"] = responses_df["Response"].apply(clean_text)
+
+        output = [str(item) for item in output]
+
+        self.data = pd.DataFrame(list(
+            zip(responses_df["Experiment"], responses_df["Question_ID"], responses_df["Item"], responses_df["Response"],output)),
+            columns=["Experiment", "Question_ID", "Item", "Response","Coding"])
+
+        return self.data
+
+    def code_results_llm(self, responses_df):
+        '''code results from LLM's response'''
+        output = []
+        '''database for Exp4'''
+        item4 = pd.read_csv(envs.ITEM_4_DATA)
+        wordpair2code = {}
+        for j in range(len(item4['Coding'])):
+            wordpair2code[item4['Pair'][j]] = item4['Coding'][j]
+        '''verb for Exp5'''
+        item5 = pd.read_csv(envs.ITEM_5_DATA)
+        # item corresponding to verb, same item id corresponding to verb pair
+        item2verb2 = {}
+        item2verb1 = {}
+
+        Stimuli1, Stimuli2 = {}, {}
+        for j in range(len(item5['Item'])):
+            item2verb1[item5['Item'][j]] = item5['Verb1'][j]
+            item2verb2[item5['Item'][j]] = item5['Verb2'][j]
+            Stimuli1[item5['ID'][j]] = item5['Stimuli-1'][j]
+            Stimuli2[item5['ID'][j]] = item5['Stimuli-2'][j]
+
+        male_keyword = ["he", "his", "himself"]
+        female_keyword = ["she", "her", "herself"]
+        #print(len(responses_df["Experiment"]))
+        for i in range(len(responses_df["Experiment"])):
+
+
+            print(i, "/", len(responses_df["Experiment"]))
+            # vote_1_1, vote_1_2, vote_1_3 = 0, 0, 0
+            # print()
+            if pd.isna(responses_df["Response"][i]):
+                output.append("Other")
+                continue
+            rs = responses_df["Response"][i].strip().lower()
+            print(rs)
+            rs = rs.replace('"', '').replace("  ", " ").replace('.', '')
+            #lines = rs.split("\n")
+            #filtered_lines = [line for line in lines if line and not (line.endswith(":") or line.endswith("："))]
+            # filtered_lines = [r.split(':', 1)[-1].strip() if ':' in r else r for
+            #       r in filtered_lines]
+            # rs = "\n".join(filtered_lines)
+            # rs = rs.strip()
+            '''Exp1'''
+            if responses_df["Experiment"][i] == "E1":
+                rs_lower = rs.lower()
+                if "round" in rs_lower and "spiky" in rs_lower:
+                    output.append("Other")
+                elif "round" in rs_lower:
+                    output.append("Round")
+                elif "spiky" in rs_lower:
+                    output.append("Spiky")
+                else:
+                    output.append("Other")
+
+                '''Exp2'''
+
+            elif responses_df["Experiment"][i] == "E2":
+                # rs = responses_df["Response"][i].strip()
+                rs = rs.split(' ')
+                #print("E2", rs)
+                male, female = 0, 0
+                for word in rs:
+                    if word in female_keyword and male == 0:
+                        female = 1
+                        output.append("Female")
+                        break
+                    if word in male_keyword and female == 0:
+                        male = 1
+                        output.append("Male")
+                        break
+                if male == 0 and female == 0:
+                    output.append("Other")
 
+                '''Exp3'''
+            elif responses_df["Experiment"][i] == "E3":
+                # rs = responses_df["Response"][i].strip()
+                #print("E3", rs)
+                pair = responses_df["Factor 2"][i]
+                word1, word2 = pair.replace(".", "").split('_')
+
+                if responses_df["Item"][i] == 12:
+                    output.append("Other")
+                else:
+                    words = rs.split()  # split the response into words
+                    if any(word == word1 for word in words) and any(word == word2 for word in words):
+                        output.append("Other")
+                    else:
+                        if any(word.lower() == word1.lower() for word in words):
+                            if len(word1) > len(word2):
+                                output.append("Long")
+                            else:
+                                output.append("Short")
+                        elif any(word.lower() == word2.lower() for word in words):
+                            if len(word1) > len(word2):
+                                output.append("Short")
+                            else:
+                                output.append("Long")
+                        else:
+                            if len(words) > 1:
+                                # joint the words using " "
+                                word = " ".join(words)
+                                if word.lower() == word1.lower():
+                                    if len(word1) > len(word2):
+                                        output.append("Long")
+                                    else:
+                                        output.append("Short")
+                                elif word.lower() == word2.lower():
+                                    if len(word1) > len(word2):
+                                        output.append("Short")
+                                    else:
+                                        output.append("Long")
+                                else:
+                                    output.append("Other")
+                            else:
+                                output.append("Other")
+
+
+                '''Exp4'''
+
+            elif responses_df["Experiment"][i] == "E4":
+                lines = rs.split("\n")
+                filtered_lines = []
+                if len(lines) > 1:
+                    for r in lines[1:]:
+                        if ':' in r:
+                            filtered_lines.append(r.split(':', 1)[-1].strip())
+                        else:
+                            filtered_lines.append(r)
+                    filtered_lines.insert(0, lines[0])
+                else:
+                    filtered_lines = lines
+                # print(filtered_lines)
+
+                #filtered_lines = [r.split('-', 1)[-1].strip() if '-' in r else r for r in filtered_lines]
+                #rs = "\n".join(filtered_lines)
+
+                #filtered_lines = [r.split(':', 1)[-1].strip() if ':' in r else r for r in rs.split(";")]
+                #filtered_lines = [r.split('-', 1)[-1].strip() if '-' in r else r for r in filtered_lines]
+                rs = ";".join(filtered_lines).strip()
+                try:
+                    meaning_word = rs.split(";")[4].replace(" ", '')
+                except IndexError:
+                    try:
+                        meaning_word = rs.split("\n")[4].replace(" ", '')
+                    except IndexError:
+                        output.append("Other")
+                        continue
+                except Exception as e:
+                    print(f"Unexpected error: {e}")
+                    output.append("Other")
+                    continue
+
+                target = responses_df["Factor 2"][i].strip().lower()
+                pair = target + "_" + meaning_word
+                #print("E4:", pair)
+
+                if pair in wordpair2code.keys():
+                    output.append(wordpair2code[pair])
+                else:
+                    output.append("Other")
+
+                '''Exp5'''
+            elif responses_df["Experiment"][i] == "E5" or responses_df["Experiment"][i] == "E51":
+                # sentence = responses_df["Response"][i].strip()
+                item_id = responses_df["Item"][i]
+                question_id = responses_df["Question_ID"][i]
+
+                if responses_df["Experiment"][i] == "E51":
+                    sti1 = Stimuli1[question_id[0:-1]].lower().replace("...", "")
+                    #sti2 = Stimuli2[question_id[0:-1]].lower().replace("...", "")
+                    verb = item2verb1[item_id].lower()
+
+                    sentence = sti1 + " " + rs.replace(sti1, "")
+                    #print("E5", verb, sentence)
+                if responses_df["Experiment"][i] == "E5":
+                    #sti1 = Stimuli1[question_id].lower().replace("...", "")
+                    # print(sti1)
+                    sti2 = Stimuli2[question_id].lower().replace("...", "")
+
+                    verb = item2verb2[item_id].lower()
+                    sentence = sti2 + " " + rs.replace(sti2, "")
+                    #print("E5", verb, sentence)
+
+                doc = nlp1(sentence.replace("  ", " "))
+                # print(doc)
+                # print()
+                verb_token = None
+                for token in doc:
+                    # print(token.lemma_)
+                    if token.lemma_ == verb:
+                        verb_token = token
+                        break
+                # exit()
+                pobj, dative = None, None
+                # print(verb_token.children)
+                # exit()
+                if verb_token is not None:
+                    for child in verb_token.children:
+                        # print(child)
+                        if (child.dep_ == 'dative' and child.pos_ == "ADP") or (
+                                child.text == "to" and child.dep_ == 'prep' and child.pos_ == "ADP"):
+                            pobj = child.text
+                        if child.dep_ == 'dative':
+                            dative = child.text
+
+                # print("E5", pobj, dative)
+                # exit()
+
+                if pobj:
+                    output.append("PO")
+                elif dative:
+                    output.append("DO")
+                else:
+                    # print("Other", sentence, pobj, dative)
+                    # exit()
+                    output.append("Other")
+
+
+
+                '''Exp6'''
+
+            elif responses_df["Experiment"][i] == "E6":
+                sentence = responses_df["Stimuli 1"][i].strip().lower()
+                #print("E6", sentence)
+                doc = nlp1(sentence)
+                subject = "None"
+                obj = "None"
+                pobj_list = []  # To collect all prepositional objects
 
                 for token in doc:
                     if token.dep_ == "nsubj":
                         subject = token.text
                     elif token.dep_ == "dobj":
                         obj = token.text
-                #print("E6", subject, obj)
-                if subject in rs and obj in rs:
-                    #print(rs, subject, obj, "Other")
+                    elif token.dep_ == "pobj":
+                        pobj_list.append(token.text)  # Collect prepositional objects
+
+                rs_list = rs.lower().split()
+                if subject in rs_list and (obj in rs_list or any(pobj == r for pobj in pobj_list for r in rs_list)):
                     output.append("Other")
-                elif subject in rs:
-                    #print(rs, subject, obj, "VP")
+                elif subject in rs_list:
                     output.append("VP")
-                elif obj in rs:
-                    #print(rs, subject, obj, "NP")
+                elif obj in rs_list or any(pobj == r for pobj in pobj_list for r in rs_list):
                     output.append("NP")
                 else:
-                    #print(rs, subject, obj, "Other")
                     output.append("Other")
 
                 '''Exp7'''
@@ -834,11 +1177,22 @@ class EvaluationModel:
         # exit()
                 '''LLM'''
         print(len(output))
+        import re
+        def clean_text(text):
+            if isinstance(text, str):
+                return re.sub(r'[^\x00-\x7F]+', '', text)
+            return text
+
+        responses_df["Experiment"] = responses_df["Experiment"].apply(clean_text)
+        responses_df["Question_ID"] = responses_df["Question_ID"].apply(clean_text)
+        responses_df["Item"] = responses_df["Item"].apply(clean_text)
+        responses_df["Response"] = responses_df["Response"].apply(clean_text)
+
+        output = [str(item) for item in output]
+
         self.data = pd.DataFrame(list(
-            zip(responses_df["Experiment"], responses_df["Question_ID"], responses_df["Item"], responses_df["Response"],
-                responses_df["Factor 2"], responses_df["Stimuli 1"], output)),
-            columns=["Experiment", "Question_ID", "Item", "Response", "Factor 2", "Stimuli 1",
-                     "Coding"])
+            zip(responses_df["Experiment"], responses_df["Question_ID"], responses_df["Item"], responses_df["Response"],output)),
+            columns=["Experiment", "Question_ID", "Item", "Response","Coding"])
 
         return self.data
 
@@ -848,6 +1202,8 @@ class EvaluationModel:
 
 
 
+
+
     def calculate_js_divergence(self, file_path_1, file_path_2):
         """
         Calculate the Jensen-Shannon divergence for response distributions between two datasets.
@@ -855,7 +1211,7 @@ class EvaluationModel:
         removes the original E5 and E51, and then calculates the JS divergence between the datasets.
 
         Parameters:
-        file_path_1 (str): Path to the first dataset file (Excel format).
+        file_path_1 (str): Path to the first dataset file (CSV format).
         file_path_2 (str): Path to the second dataset file (CSV format).
 
         Returns:
@@ -893,17 +1249,15 @@ class EvaluationModel:
         human_df = pd.concat([human_df, human_e5], ignore_index=True)
         llm_df = pd.concat([llm_df, llm_e5], ignore_index=True)
 
-
         ### Calculate Average JS Divergence ###
 
-
         # Extract the relevant columns for JS divergence calculation
         human_responses = human_df[['Question_ID', 'Coding']]
         llm_responses = llm_df[['Question_ID', 'Coding']]
 
         # Remove 'Other' responses
-        human_responses = human_responses[human_responses['Coding'] != 'Other']
-        llm_responses = llm_responses[llm_responses['Coding'] != 'Other']
+        #human_responses = human_responses[human_responses['Coding'] != 'Other']
+        #llm_responses = llm_responses[llm_responses['Coding'] != 'Other']
 
         # Get unique Question_IDs present in both datasets
         common_question_ids = set(human_responses['Question_ID']).intersection(set(llm_responses['Question_ID']))
@@ -933,6 +1287,7 @@ class EvaluationModel:
 
         # Calculate the average JS divergence per experiment and the confidence interval
         results = {}
+        experiment_averages = []
         for exp, divs in js_divergence.items():
             avg_js_divergence = 1 - np.nanmean(divs)
             ci_lower, ci_upper = bootstrap((divs,), np.nanmean, confidence_level=0.95,
@@ -941,14 +1296,14 @@ class EvaluationModel:
                 'average_js_divergence': avg_js_divergence,
                 'confidence_interval': (1 - ci_upper, 1 - ci_lower)  # Adjust for 1 - score
             }
+            experiment_averages.append(avg_js_divergence)
 
-        # Calculate the overall average JS divergence and confidence interval
-        overall_js_divergence = 1 - np.nanmean([js for divs in js_divergence.values() for js in divs])
-        flattened_js_divergence = np.concatenate([np.array(divs) for divs in js_divergence.values()])
+        # Calculate the weighted average JS divergence across all experiments
+        weighted_js_divergence = np.mean(experiment_averages)  # Simple average over experiments
 
-        # 计算总体的置信区间
+        # Calculate the confidence interval for the overall JS divergence using bootstrap
         overall_ci_lower, overall_ci_upper = bootstrap(
-            (flattened_js_divergence,),
+            (experiment_averages,),
             np.nanmean,
             confidence_level=0.95,
             n_resamples=1000
@@ -957,8 +1312,8 @@ class EvaluationModel:
         # Combine all results into one dictionary
         all_results = {
             'overall': {
-                'average_js_divergence': overall_js_divergence,
-                'confidence_interval': (1 - overall_ci_upper, 1 - overall_ci_lower)
+                'average_js_divergence': weighted_js_divergence,
+                'confidence_interval': (overall_ci_lower, overall_ci_upper)
             },
             'per_experiment': results
         }