RichardErkhov commited on
Commit
7bf2053
·
verified ·
1 Parent(s): 5882a58

uploaded model

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +63 -0
special_tokens_map.json ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<nexa_0>",
4
+ "<nexa_1>",
5
+ "<nexa_2>",
6
+ "<nexa_3>",
7
+ "<nexa_4>",
8
+ "<nexa_5>",
9
+ "<nexa_6>",
10
+ "<nexa_7>",
11
+ "<nexa_8>",
12
+ "<nexa_9>",
13
+ "<nexa_10>",
14
+ "<nexa_11>",
15
+ "<nexa_12>",
16
+ "<nexa_13>",
17
+ "<nexa_14>",
18
+ "<nexa_15>",
19
+ "<nexa_16>",
20
+ "<nexa_17>",
21
+ "<nexa_18>",
22
+ "<nexa_19>",
23
+ "<nexa_20>",
24
+ "<nexa_21>",
25
+ "<nexa_22>",
26
+ "<nexa_23>",
27
+ "<nexa_24>",
28
+ "<nexa_25>",
29
+ "<nexa_26>",
30
+ "<nexa_27>",
31
+ "<nexa_28>",
32
+ "<nexa_29>",
33
+ "<nexa_end>"
34
+ ],
35
+ "bos_token": {
36
+ "content": "<s>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false
41
+ },
42
+ "eos_token": {
43
+ "content": "<|endoftext|>",
44
+ "lstrip": false,
45
+ "normalized": false,
46
+ "rstrip": false,
47
+ "single_word": false
48
+ },
49
+ "pad_token": {
50
+ "content": "<|endoftext|>",
51
+ "lstrip": false,
52
+ "normalized": false,
53
+ "rstrip": false,
54
+ "single_word": false
55
+ },
56
+ "unk_token": {
57
+ "content": "<unk>",
58
+ "lstrip": false,
59
+ "normalized": false,
60
+ "rstrip": false,
61
+ "single_word": false
62
+ }
63
+ }