Commit
·
1b20edf
1
Parent(s):
eea2f56
fix eoa and eoh
Browse files
modeling_InternLM_XComposer.py
CHANGED
@@ -101,15 +101,8 @@ conversation
|
|
101 |
|
102 |
self.tokenizer = None
|
103 |
|
104 |
-
|
105 |
-
|
106 |
-
return self.tokenizer.decode(torch.Tensor([103027]),
|
107 |
-
skip_special_tokens=True)
|
108 |
-
|
109 |
-
@property
|
110 |
-
def eoa(self):
|
111 |
-
return self.tokenizer.decode(torch.Tensor([103028]),
|
112 |
-
skip_special_tokens=True)
|
113 |
|
114 |
def maybe_autocast(self, dtype=torch.float16):
|
115 |
# if on cpu, don't use autocast
|
|
|
101 |
|
102 |
self.tokenizer = None
|
103 |
|
104 |
+
self.eoh = '<TOKENS_UNUSED_0>' # end of human
|
105 |
+
self.eoa = '<TOKENS_UNUSED_1>' # end of assistant
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
106 |
|
107 |
def maybe_autocast(self, dtype=torch.float16):
|
108 |
# if on cpu, don't use autocast
|