|
{ |
|
"version": "1.0", |
|
"truncation": null, |
|
"padding": null, |
|
"added_tokens": [ |
|
{ |
|
"id": 0, |
|
"content": "[PAD]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 1, |
|
"content": "[UNK]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 2, |
|
"content": "[CLS]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 3, |
|
"content": "[SEP]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 4, |
|
"content": "[MASK]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
} |
|
], |
|
"normalizer": { |
|
"type": "Sequence", |
|
"normalizers": [ |
|
{ |
|
"type": "NFC" |
|
}, |
|
{ |
|
"type": "Lowercase" |
|
} |
|
] |
|
}, |
|
"pre_tokenizer": { |
|
"type": "Whitespace" |
|
}, |
|
"post_processor": { |
|
"type": "TemplateProcessing", |
|
"single": [ |
|
{ |
|
"SpecialToken", |
|
"id": "[CLS]" |
|
}, |
|
{ |
|
"Sequence", |
|
"id": "A" |
|
}, |
|
{ |
|
"SpecialToken", |
|
"id": "[SEP]" |
|
} |
|
], |
|
"pair": [ |
|
{ |
|
"SpecialToken", |
|
"id": "[CLS]" |
|
}, |
|
{ |
|
"Sequence", |
|
"id": "A" |
|
}, |
|
{ |
|
"SpecialToken", |
|
"id": "[SEP]" |
|
}, |
|
{ |
|
"Sequence", |
|
"id": "B" |
|
}, |
|
{ |
|
"SpecialToken", |
|
"id": "[SEP]" |
|
} |
|
], |
|
"special_tokens": { |
|
"[CLS]": { |
|
"id": "[CLS]", |
|
"ids": [2] |
|
}, |
|
"[SEP]": { |
|
"id": "[SEP]", |
|
"ids": [3] |
|
} |
|
} |
|
}, |
|
"decoder": { |
|
"type": "WordPiece", |
|
"prefix": "##", |
|
"cleanup": true |
|
}, |
|
"model": { |
|
"type": "WordPiece", |
|
"unk_token": "[UNK]", |
|
"continuing_subword_prefix": "##", |
|
"max_input_chars_per_word": 100, |
|
"vocab": { |
|
"[PAD]": 0, |
|
"[UNK]": 1, |
|
"[CLS]": 2, |
|
"[SEP]": 3, |
|
"[MASK]": 4, |
|
"the": 5, |
|
"quick": 6, |
|
"brown": 7, |
|
"fox": 8, |
|
"jumps": 9, |
|
"over": 10, |
|
"lazy": 11, |
|
"dog": 12 |
|
} |
|
} |
|
} |