Uhhy commited on
Commit
16e52aa
·
verified ·
1 Parent(s): b521fb8

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +6 -1
app.py CHANGED
@@ -110,7 +110,12 @@ def remove_duplicates(text):
110
  text = re.sub(r'(How are you\? \[/INST\]){2,}', 'How are you? [/INST]', text)
111
  text = text.replace('[/INST]', '')
112
  lines = text.split('\n')
113
- unique_lines = list(dict.fromkeys(lines))
 
 
 
 
 
114
  return '\n'.join(unique_lines)
115
 
116
  def remove_repetitive_responses(responses):
 
110
  text = re.sub(r'(How are you\? \[/INST\]){2,}', 'How are you? [/INST]', text)
111
  text = text.replace('[/INST]', '')
112
  lines = text.split('\n')
113
+ unique_lines = []
114
+ seen_lines = set()
115
+ for line in lines:
116
+ if line not in seen_lines:
117
+ seen_lines.add(line)
118
+ unique_lines.append(line)
119
  return '\n'.join(unique_lines)
120
 
121
  def remove_repetitive_responses(responses):