add tokenizer

runs/May06_13-16-26_hydcorp-esxi09/1651823248.3015673/events.out.tfevents.1651823248.hydcorp-esxi09.1528033.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0ffbe08ee037c796340e8508e7e9bad27630feaa41a49288f2e1bf4cf442a3c
+size 4786

runs/May06_13-16-26_hydcorp-esxi09/events.out.tfevents.1651823248.hydcorp-esxi09.1528033.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:576faa20ea3c738d5dc0bb38a4fefe877111107b18872f9240e81b5b84d267ac
+size 4136

vocab.json

@@ -1 +1 @@
-{"o": 0, "d": 1, "a": 2, "e": 3, "h": 4, "r": 6, "q": 7, "n": 8, "i": 9, "p": 10, "'": 11, "v": 12, "y": 13, "b": 14, "m": 15, "x": 16, "z": 17, "u": 18, "c": 19, "w": 20, "l": 21, "s": 22, "j": 23, "t": 24, "k": 25, "g": 26, "f": 27, "|": 5, "[UNK]": 28, "[PAD]": 29}
+{"3": 0, "i": 1, "k": 2, "e": 3, "l": 4, "'": 5, "5": 6, "h": 7, "y": 8, "z": 9, "u": 10, "j": 11, "0": 12, "q": 13, "2": 14, "b": 15, "s": 16, "g": 17, "8": 18, "n": 19, "c": 20, "v": 21, "m": 22, "4": 23, "r": 24, "d": 25, "w": 26, "a": 27, "o": 29, "6": 30, "x": 31, "9": 32, "t": 33, "f": 34, "1": 35, "p": 36, "|": 28, "[UNK]": 37, "[PAD]": 38}