



'''simple docstring'''






import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
						UpperCamelCase__:  int				      =     "pt"
elif is_tf_available():
						UpperCamelCase__:  List[Any]				      =     "tf"
else:
						UpperCamelCase__:  int				      =     "jax"

class 							SCREAMING_SNAKE_CASE(				SCREAMING_SNAKE_CASE__				,						unittest.TestCase     ):







	"""simple docstring"""






	lowerCamelCase__     					=			ByTaTokenizer
	lowerCamelCase__     					=			False







	def 	A      (		self     :	Union[str, Any]						)     ->						Any:
								super().setUp()
								UpperCAmelCase					:			int									=						ByTaTokenizer()
								tokenizer.save_pretrained(self.tmpdirname						)







	@cached_property
	def 	A      (		self     :	Tuple						)     ->						Union[str, Any]:
								return ByTaTokenizer.from_pretrained('''google/byt5-small'''						)







	def 	A      (		self     :	Optional[int]						,     **__snake_case     :	Any						)     ->						ByTaTokenizer:
								return self.tokenizer_class.from_pretrained(self.tmpdirname						,     **__snake_case						)







	def 	A      (		self     :	Any						,     __snake_case     :	Union[str, Any]						,     __snake_case     :	List[Any]=False						,     __snake_case     :	Optional[int]=20						,     __snake_case     :	Optional[int]=5						)     ->						Tuple[str, list]:
								UpperCAmelCase					:			Dict									=						[]
								for i in range(len(__snake_case						)						):
															try:
																						UpperCAmelCase					:			Optional[int]									=						tokenizer.decode([i]						,     clean_up_tokenization_spaces=__snake_case						)
															except UnicodeDecodeError:
																						pass
															toks.append((i, tok)						)

								UpperCAmelCase					:			Tuple									=						list(filter(lambda __snake_case						: re.match(r'''^[ a-zA-Z]+$'''						,     t[1]						)						,     __snake_case						)						)
								UpperCAmelCase					:			str									=						list(filter(lambda __snake_case						: [t[0]] == tokenizer.encode(t[1]						,     add_special_tokens=__snake_case						)						,     __snake_case						)						)
								if max_length is not None and len(__snake_case						) > max_length:
															UpperCAmelCase					:			Dict									=						toks[:max_length]
								if min_length is not None and len(__snake_case						) < min_length and len(__snake_case						) > 0:
															while len(__snake_case						) < min_length:
																						UpperCAmelCase					:			Dict									=						toks + toks
        # toks_str = [t[1] for t in toks]
								UpperCAmelCase					:			Optional[Any]									=						[t[0] for t in toks]

								# Ensure consistency
								UpperCAmelCase					:			int									=						tokenizer.decode(__snake_case						,     clean_up_tokenization_spaces=__snake_case						)
								if " " not in output_txt and len(__snake_case						) > 1:
															UpperCAmelCase					:			Union[str, Any]									=						(
															    tokenizer.decode([toks_ids[0]]						,     clean_up_tokenization_spaces=__snake_case						)
															    + ''' '''
															    + tokenizer.decode(toks_ids[1:]						,     clean_up_tokenization_spaces=__snake_case						)
															)
								if with_prefix_space:
															UpperCAmelCase					:			Tuple									=						''' ''' + output_txt
								UpperCAmelCase					:			Dict									=						tokenizer.encode(__snake_case						,     add_special_tokens=__snake_case						)
								return output_txt, output_ids







	def 	A      (		self     :	Optional[int]						)     ->						Optional[Any]:
								UpperCAmelCase					:			str									=						self.ta_base_tokenizer
								UpperCAmelCase					:			Optional[int]									=						tokenizer(['''hi</s>''', '''I went to the gym</s>''', '''</s>''']						)
								UpperCAmelCase					:			int									=						tokenizer(['''hi''', '''I went to the gym''', '''''']						)
								self.assertListEqual(batch_with_eos_added['''input_ids''']						,     batch_without_eos_added['''input_ids''']						)







	def 	A      (		self     :	List[str]						)     ->						List[str]:
								UpperCAmelCase					:			Optional[Any]									=						self.ta_base_tokenizer
								UpperCAmelCase					:			Dict									=						'''Unicode €.'''
								UpperCAmelCase					:			Union[str, Any]									=						tokenizer(__snake_case						)
								UpperCAmelCase					:			Union[str, Any]									=						[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
								self.assertEqual(encoded['''input_ids''']						,     __snake_case						)

								# decoding
								UpperCAmelCase					:			Any									=						tokenizer.decode(__snake_case						)
								self.assertEqual(__snake_case						,     '''Unicode €.</s>'''						)

								UpperCAmelCase					:			Optional[int]									=						tokenizer('''e è é ê ë'''						)
								UpperCAmelCase					:			str									=						[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
								self.assertEqual(encoded['''input_ids''']						,     __snake_case						)
								# decoding
								UpperCAmelCase					:			Any									=						tokenizer.decode(__snake_case						)
								self.assertEqual(__snake_case						,     '''e è é ê ë</s>'''						)

								# encode/decode, but with `encode` instead of `__call__`
								self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''						)						)						,     '''e è é ê ë</s>'''						)







	def 	A      (		self     :	int						)     ->						Tuple:
								UpperCAmelCase					:			int									=						self.ta_base_tokenizer
								UpperCAmelCase					:			Optional[int]									=						['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
								# fmt: off
								UpperCAmelCase					:			str									=						[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
								# fmt: on
								UpperCAmelCase					:			Optional[int]									=						tokenizer(__snake_case						,     padding=__snake_case						,     return_tensors=__snake_case						)
								self.assertIsInstance(__snake_case						,     __snake_case						)

								if FRAMEWORK != "jax":
															UpperCAmelCase					:			Dict									=						list(batch.input_ids.numpy()[0]						)
								else:
															UpperCAmelCase					:			Any									=						list(batch.input_ids.tolist()[0]						)

								self.assertListEqual(__snake_case						,     __snake_case						)

								self.assertEqual((2, 37)						,     batch.input_ids.shape						)
								self.assertEqual((2, 37)						,     batch.attention_mask.shape						)







	def 	A      (		self     :	Any						)     ->						Tuple:
								UpperCAmelCase					:			int									=						self.ta_base_tokenizer
								UpperCAmelCase					:			Dict									=						['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
								UpperCAmelCase					:			Tuple									=						tokenizer(__snake_case						,     padding=__snake_case						,     return_tensors=__snake_case						)
								# check if input_ids are returned and no decoder_input_ids
								self.assertIn('''input_ids'''						,     __snake_case						)
								self.assertIn('''attention_mask'''						,     __snake_case						)
								self.assertNotIn('''decoder_input_ids'''						,     __snake_case						)
								self.assertNotIn('''decoder_attention_mask'''						,     __snake_case						)







	def 	A      (		self     :	Optional[int]						)     ->						List[Any]:
								UpperCAmelCase					:			int									=						self.ta_base_tokenizer
								UpperCAmelCase					:			List[str]									=						[
								    '''Summary of the text.''',
								    '''Another summary.''',
								]
								UpperCAmelCase					:			List[str]									=						tokenizer(
								    text_target=__snake_case						,     max_length=32						,     padding='''max_length'''						,     truncation=__snake_case						,     return_tensors=__snake_case						)
								self.assertEqual(32						,     targets['''input_ids'''].shape[1]						)







	def 	A      (		self     :	Optional[Any]						)     ->						Tuple:
								UpperCAmelCase					:			str									=						self.ta_base_tokenizer
								UpperCAmelCase					:			Optional[int]									=						['''A long paragraph for summarization. </s>''']
								UpperCAmelCase					:			str									=						['''Summary of the text. </s>''']
								# fmt: off
								UpperCAmelCase					:			Optional[int]									=						[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
								UpperCAmelCase					:			int									=						[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
								# fmt: on

								UpperCAmelCase					:			str									=						tokenizer(__snake_case						,     text_target=__snake_case						)

								self.assertEqual(__snake_case						,     batch['''input_ids'''][0]						)
								self.assertEqual(__snake_case						,     batch['''labels'''][0]						)







	def 	A      (		self     :	str						)     ->						Dict:
								UpperCAmelCase					:			int									=						self.get_tokenizers()
								for tokenizer in tokenizers:
															with self.subTest(F"""{tokenizer.__class__.__name__}"""						):
																						self.assertNotEqual(tokenizer.model_max_length						,     42						)

        # Now let's start the test
								UpperCAmelCase					:			List[str]									=						self.get_tokenizers()
								for tokenizer in tokenizers:
															with self.subTest(F"""{tokenizer.__class__.__name__}"""						):
																						# Isolate this from the other tests because we save additional tokens/etc
																						UpperCAmelCase					:			Union[str, Any]									=						tempfile.mkdtemp()

																						UpperCAmelCase					:			Any									=						''' He is very happy, UNwant\u00E9d,running'''
																						UpperCAmelCase					:			Optional[Any]									=						tokenizer.encode(__snake_case						,     add_special_tokens=__snake_case						)
																						tokenizer.save_pretrained(__snake_case						)

																						UpperCAmelCase					:			Dict									=						tokenizer.__class__.from_pretrained(__snake_case						)
																						UpperCAmelCase					:			Optional[int]									=						after_tokenizer.encode(__snake_case						,     add_special_tokens=__snake_case						)
																						self.assertListEqual(__snake_case						,     __snake_case						)

																						shutil.rmtree(__snake_case						)

								UpperCAmelCase					:			int									=						self.get_tokenizers(model_max_length=42						)
								for tokenizer in tokenizers:
															with self.subTest(F"""{tokenizer.__class__.__name__}"""						):
																						# Isolate this from the other tests because we save additional tokens/etc
																						UpperCAmelCase					:			List[Any]									=						tempfile.mkdtemp()

																						UpperCAmelCase					:			Optional[int]									=						''' He is very happy, UNwant\u00E9d,running'''
																						tokenizer.add_tokens(['''bim''', '''bambam''']						)
																						UpperCAmelCase					:			Tuple									=						tokenizer.additional_special_tokens
																						additional_special_tokens.append('''new_additional_special_token'''						)
																						tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}						)
																						UpperCAmelCase					:			str									=						tokenizer.encode(__snake_case						,     add_special_tokens=__snake_case						)
																						tokenizer.save_pretrained(__snake_case						)

																						UpperCAmelCase					:			List[Any]									=						tokenizer.__class__.from_pretrained(__snake_case						)
																						UpperCAmelCase					:			Optional[int]									=						after_tokenizer.encode(__snake_case						,     add_special_tokens=__snake_case						)
																						self.assertListEqual(__snake_case						,     __snake_case						)
																						self.assertIn('''new_additional_special_token'''						,     after_tokenizer.additional_special_tokens						)
																						self.assertEqual(after_tokenizer.model_max_length						,     42						)

																						UpperCAmelCase					:			str									=						tokenizer.__class__.from_pretrained(__snake_case						,     model_max_length=43						)
																						self.assertEqual(tokenizer.model_max_length						,     43						)

																						shutil.rmtree(__snake_case						)







	def 	A      (		self     :	List[Any]						)     ->						str:
								UpperCAmelCase					:			Optional[Any]									=						[]
								if self.test_slow_tokenizer:
															tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

								if self.test_rust_tokenizer:
															tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

								for tokenizer_class, tokenizer_utils in tokenizer_list:
															with tempfile.TemporaryDirectory() as tmp_dir:
																						tokenizer_utils.save_pretrained(__snake_case						)

																						with open(os.path.join(__snake_case						,     '''special_tokens_map.json'''						)						,     encoding='''utf-8'''						) as json_file:
																													UpperCAmelCase					:			Optional[int]									=						json.load(__snake_case						)

																						with open(os.path.join(__snake_case						,     '''tokenizer_config.json'''						)						,     encoding='''utf-8'''						) as json_file:
																													UpperCAmelCase					:			Union[str, Any]									=						json.load(__snake_case						)

																						UpperCAmelCase					:			Tuple									=						[F"""<extra_id_{i}>""" for i in range(125						)]

																						UpperCAmelCase					:			Any									=						added_tokens_extra_ids + [
																						    '''an_additional_special_token'''
																						]
																						UpperCAmelCase					:			Dict									=						added_tokens_extra_ids + [
																						    '''an_additional_special_token'''
																						]

																						with open(os.path.join(__snake_case						,     '''special_tokens_map.json'''						)						,     '''w'''						,     encoding='''utf-8'''						) as outfile:
																													json.dump(__snake_case						,     __snake_case						)
																						with open(os.path.join(__snake_case						,     '''tokenizer_config.json'''						)						,     '''w'''						,     encoding='''utf-8'''						) as outfile:
																													json.dump(__snake_case						,     __snake_case						)

																						# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																						# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																						# "special_tokens_map.json" files
																						UpperCAmelCase					:			List[str]									=						tokenizer_class.from_pretrained(
																						    __snake_case						,     )
																						self.assertIn(
																						    '''an_additional_special_token'''						,     tokenizer_without_change_in_init.additional_special_tokens						)
																						# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																						self.assertEqual(
																						    ['''an_additional_special_token''']						,     tokenizer_without_change_in_init.convert_ids_to_tokens(
																						        tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']						)						)						,     )

																						# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																						UpperCAmelCase					:			Dict									=						added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''						,     lstrip=__snake_case						)]
																						UpperCAmelCase					:			Union[str, Any]									=						tokenizer_class.from_pretrained(
																						    __snake_case						,     additional_special_tokens=__snake_case						,     )

																						self.assertIn('''a_new_additional_special_token'''						,     tokenizer.additional_special_tokens						)
																						self.assertEqual(
																						    ['''a_new_additional_special_token''']						,     tokenizer.convert_ids_to_tokens(
																						        tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']						)						)						,     )







	def 	A      (		self     :	Optional[Any]						)     ->						List[str]:
								UpperCAmelCase					:			int									=						[]
								if self.test_slow_tokenizer:
															tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

								if self.test_rust_tokenizer:
															tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

								for tokenizer_class, tokenizer_utils in tokenizer_list:
															with tempfile.TemporaryDirectory() as tmp_dir:
																						tokenizer_utils.save_pretrained(__snake_case						)

																						UpperCAmelCase					:			Dict									=						tokenizer_class.from_pretrained(__snake_case						)

																						self.assertTrue(tokenizer.decode([255]						) == ''''''						)







	def 	A      (		self     :	Any						)     ->						Union[str, Any]:
								pass







	def 	A      (		self     :	Dict						)     ->						Dict:
								pass







	def 	A      (		self     :	Tuple						)     ->						Any:
								pass







	def 	A      (		self     :	List[Any]						)     ->						int:
								pass







	def 	A      (		self     :	Union[str, Any]						)     ->						Tuple:
								UpperCAmelCase					:			List[str]									=						self.get_tokenizers(fast=__snake_case						,     do_lower_case=__snake_case						)
								for tokenizer in tokenizers:
															with self.subTest(F"""{tokenizer.__class__.__name__}"""						):
																						UpperCAmelCase					:			Optional[Any]									=						['''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''x''', '''t''', '''</s>''']
																						UpperCAmelCase					:			Union[str, Any]									=						tokenizer.convert_tokens_to_string(__snake_case						)

																						self.assertIsInstance(__snake_case						,     __snake_case						)







	def 	A      (		self     :	Any						)     ->						Tuple:
								UpperCAmelCase					:			Dict									=						self.get_tokenizers()
								for tokenizer in tokenizers:
															with self.subTest(F"""{tokenizer.__class__.__name__}"""						):
																						UpperCAmelCase					:			Optional[Any]									=						[
																						    '''bos_token''',
																						    '''eos_token''',
																						    '''unk_token''',
																						    '''sep_token''',
																						    '''pad_token''',
																						    '''cls_token''',
																						    '''mask_token''',
																						]

																						UpperCAmelCase					:			Optional[int]									=						0
																						UpperCAmelCase					:			Any									=						tokenizer.convert_ids_to_tokens(
																						    __snake_case						,     skip_special_tokens=__snake_case						)

																						for attr in attributes_list:
																													setattr(__snake_case						,     attr + '''_id'''						,     __snake_case						)
																													self.assertEqual(getattr(__snake_case						,     __snake_case						)						,     __snake_case						)
																													self.assertEqual(getattr(__snake_case						,     attr + '''_id'''						)						,     __snake_case						)

																													setattr(__snake_case						,     attr + '''_id'''						,     __snake_case						)
																													self.assertEqual(getattr(__snake_case						,     __snake_case						)						,     __snake_case						)
																													self.assertEqual(getattr(__snake_case						,     attr + '''_id'''						)						,     __snake_case						)

																						setattr(__snake_case						,     '''additional_special_tokens_ids'''						,     []						)
																						self.assertListEqual(getattr(__snake_case						,     '''additional_special_tokens'''						)						,     []						)
																						self.assertListEqual(getattr(__snake_case						,     '''additional_special_tokens_ids'''						)						,     []						)

																						setattr(__snake_case						,     '''additional_special_tokens_ids'''						,     [token_id_to_test_setters]						)
																						self.assertListEqual(getattr(__snake_case						,     '''additional_special_tokens'''						)						,     [token_to_test_setters]						)
																						self.assertListEqual(getattr(__snake_case						,     '''additional_special_tokens_ids'''						)						,     [token_id_to_test_setters]						)











import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


SCREAMING_SNAKE_CASE__     :     Dict		    =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     :     int		    =       {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE__     :     Dict		    =       {
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

SCREAMING_SNAKE_CASE__     :     List[Any]		    =       {
    "bert-base-uncased": 5_12,
    "bert-large-uncased": 5_12,
    "bert-base-cased": 5_12,
    "bert-large-cased": 5_12,
    "bert-base-multilingual-uncased": 5_12,
    "bert-base-multilingual-cased": 5_12,
    "bert-base-chinese": 5_12,
    "bert-base-german-cased": 5_12,
    "bert-large-uncased-whole-word-masking": 5_12,
    "bert-large-cased-whole-word-masking": 5_12,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 5_12,
    "bert-large-cased-whole-word-masking-finetuned-squad": 5_12,
    "bert-base-cased-finetuned-mrpc": 5_12,
    "bert-base-german-dbmdz-cased": 5_12,
    "bert-base-german-dbmdz-uncased": 5_12,
    "TurkuNLP/bert-base-finnish-cased-v1": 5_12,
    "TurkuNLP/bert-base-finnish-uncased-v1": 5_12,
    "wietsedv/bert-base-dutch-cased": 5_12,
}

SCREAMING_SNAKE_CASE__     :     Union[str, Any]		    =       {
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}



class       a_       (     SCREAMING_SNAKE_CASE__     ):
	A    							= VOCAB_FILES_NAMES
	A    							= PRETRAINED_VOCAB_FILES_MAP
	A    							= PRETRAINED_INIT_CONFIGURATION
	A    							= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A    							= BertTokenizer





	def __init__(							self      ,     SCREAMING_SNAKE_CASE=None      ,     SCREAMING_SNAKE_CASE=None      ,     SCREAMING_SNAKE_CASE=True      ,     SCREAMING_SNAKE_CASE="[UNK]"      ,     SCREAMING_SNAKE_CASE="[SEP]"      ,     SCREAMING_SNAKE_CASE="[PAD]"      ,     SCREAMING_SNAKE_CASE="[CLS]"      ,     SCREAMING_SNAKE_CASE="[MASK]"      ,     SCREAMING_SNAKE_CASE=True      ,     SCREAMING_SNAKE_CASE=None      ,     **SCREAMING_SNAKE_CASE      ,     )  ->					List[str]:




				"""simple docstring"""



				super().__init__(
				    SCREAMING_SNAKE_CASE      ,     tokenizer_file=SCREAMING_SNAKE_CASE      ,     do_lower_case=SCREAMING_SNAKE_CASE      ,     unk_token=SCREAMING_SNAKE_CASE      ,     sep_token=SCREAMING_SNAKE_CASE      ,     pad_token=SCREAMING_SNAKE_CASE      ,     cls_token=SCREAMING_SNAKE_CASE      ,     mask_token=SCREAMING_SNAKE_CASE      ,     tokenize_chinese_chars=SCREAMING_SNAKE_CASE      ,     strip_accents=SCREAMING_SNAKE_CASE      ,     **SCREAMING_SNAKE_CASE      ,     )

				SCREAMING_SNAKE_CASE_								=						json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
				if (
				    normalizer_state.get('lowercase'      ,     SCREAMING_SNAKE_CASE      ) != do_lower_case
				    or normalizer_state.get('strip_accents'      ,     SCREAMING_SNAKE_CASE      ) != strip_accents
				    or normalizer_state.get('handle_chinese_chars'      ,     SCREAMING_SNAKE_CASE      ) != tokenize_chinese_chars
				):
							SCREAMING_SNAKE_CASE_								=						getattr(SCREAMING_SNAKE_CASE      ,     normalizer_state.pop('type'      )      )
							SCREAMING_SNAKE_CASE_								=						do_lower_case
							SCREAMING_SNAKE_CASE_								=						strip_accents
							SCREAMING_SNAKE_CASE_								=						tokenize_chinese_chars
							SCREAMING_SNAKE_CASE_								=						normalizer_class(**SCREAMING_SNAKE_CASE      )

				SCREAMING_SNAKE_CASE_								=						do_lower_case





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE=None      )  ->					Union[str, Any]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE = None      )  ->					List[int]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						[self.sep_token_id]
				SCREAMING_SNAKE_CASE_								=						[self.cls_token_id]
				if token_ids_a is None:
							return len(cls + token_ids_a + sep      ) * [0]
				return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE = None      )  ->					Tuple[str]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						self._tokenizer.model.save(SCREAMING_SNAKE_CASE      ,     name=SCREAMING_SNAKE_CASE      )
				return tuple(SCREAMING_SNAKE_CASE      )
