Fischerboot commited on
Commit
c1a4083
·
verified ·
1 Parent(s): 2a74bf7

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +0 -48
tokenizer_config.json CHANGED
@@ -27,54 +27,6 @@
27
  "single_word": false,
28
  "special": true
29
  },
30
- "32000": {
31
- "content": "<thinking>",
32
- "lstrip": false,
33
- "normalized": false,
34
- "rstrip": false,
35
- "single_word": false,
36
- "special": false
37
- },
38
- "32001": {
39
- "content": "</thinking>",
40
- "lstrip": false,
41
- "normalized": false,
42
- "rstrip": false,
43
- "single_word": false,
44
- "special": false
45
- },
46
- "32002": {
47
- "content": "<reflection>",
48
- "lstrip": false,
49
- "normalized": false,
50
- "rstrip": false,
51
- "single_word": false,
52
- "special": false
53
- },
54
- "32003": {
55
- "content": "</reflection>",
56
- "lstrip": false,
57
- "normalized": false,
58
- "rstrip": false,
59
- "single_word": false,
60
- "special": false
61
- },
62
- "32004": {
63
- "content": "<output>",
64
- "lstrip": false,
65
- "normalized": false,
66
- "rstrip": false,
67
- "single_word": false,
68
- "special": false
69
- },
70
- "32005": {
71
- "content": "</output>",
72
- "lstrip": false,
73
- "normalized": false,
74
- "rstrip": false,
75
- "single_word": false,
76
- "special": false
77
- }
78
  },
79
  "bos_token": "<s>",
80
  "clean_up_tokenization_spaces": false,
 
27
  "single_word": false,
28
  "special": true
29
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  },
31
  "bos_token": "<s>",
32
  "clean_up_tokenization_spaces": false,