Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -110,7 +110,12 @@ def remove_duplicates(text):
|
|
110 |
text = re.sub(r'(How are you\? \[/INST\]){2,}', 'How are you? [/INST]', text)
|
111 |
text = text.replace('[/INST]', '')
|
112 |
lines = text.split('\n')
|
113 |
-
unique_lines =
|
|
|
|
|
|
|
|
|
|
|
114 |
return '\n'.join(unique_lines)
|
115 |
|
116 |
def remove_repetitive_responses(responses):
|
|
|
110 |
text = re.sub(r'(How are you\? \[/INST\]){2,}', 'How are you? [/INST]', text)
|
111 |
text = text.replace('[/INST]', '')
|
112 |
lines = text.split('\n')
|
113 |
+
unique_lines = []
|
114 |
+
seen_lines = set()
|
115 |
+
for line in lines:
|
116 |
+
if line not in seen_lines:
|
117 |
+
seen_lines.add(line)
|
118 |
+
unique_lines.append(line)
|
119 |
return '\n'.join(unique_lines)
|
120 |
|
121 |
def remove_repetitive_responses(responses):
|