Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
|
@@ -110,7 +110,12 @@ def remove_duplicates(text):
|
|
| 110 |
text = re.sub(r'(How are you\? \[/INST\]){2,}', 'How are you? [/INST]', text)
|
| 111 |
text = text.replace('[/INST]', '')
|
| 112 |
lines = text.split('\n')
|
| 113 |
-
unique_lines =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
return '\n'.join(unique_lines)
|
| 115 |
|
| 116 |
def remove_repetitive_responses(responses):
|
|
|
|
| 110 |
text = re.sub(r'(How are you\? \[/INST\]){2,}', 'How are you? [/INST]', text)
|
| 111 |
text = text.replace('[/INST]', '')
|
| 112 |
lines = text.split('\n')
|
| 113 |
+
unique_lines = []
|
| 114 |
+
seen_lines = set()
|
| 115 |
+
for line in lines:
|
| 116 |
+
if line not in seen_lines:
|
| 117 |
+
seen_lines.add(line)
|
| 118 |
+
unique_lines.append(line)
|
| 119 |
return '\n'.join(unique_lines)
|
| 120 |
|
| 121 |
def remove_repetitive_responses(responses):
|