Ethylene commited on
Commit
d16f705
·
verified ·
1 Parent(s): 4a6db11

Fix text splitting

Browse files

previous splitting code created tokens which were unable to be processed

Files changed (1) hide show
  1. RunMiniLM.cs +1 -1
RunMiniLM.cs CHANGED
@@ -29,7 +29,7 @@ public class RunMiniLM : MonoBehaviour
29
 
30
  void Start()
31
  {
32
- tokens = vocabAsset.text.Split("\r\n");
33
 
34
  engine = CreateMLModel();
35
 
 
29
 
30
  void Start()
31
  {
32
+ tokens = vocabAsset.text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.RemoveEmptyEntries);
33
 
34
  engine = CreateMLModel();
35