
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class 			_lowerCAmelCase      (			__a      ,   unittest.TestCase ):
				_lowercase    	=BertTokenizer
				_lowercase    	=BertTokenizerFast
				_lowercase    	=True
				_lowercase    	=True
				_lowercase    	=filter_non_english




				def   __a		(      self  )       ->   Optional[int]:
							super().setUp()

							lowerCAmelCase_      			=			[
							    "[UNK]",
							    "[CLS]",
							    "[SEP]",
							    "[PAD]",
							    "[MASK]",
							    "want",
							    "##want",
							    "##ed",
							    "wa",
							    "un",
							    "runn",
							    "##ing",
							    ",",
							    "low",
							    "lowest",
							]
							lowerCAmelCase_      			=			os.path.join(self.tmpdirname       ,       VOCAB_FILES_NAMES["vocab_file"]  )
							with open(self.vocab_file       ,       "w"       ,       encoding="utf-8"  ) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]  )  )




				def   __a		(      self       ,       _UpperCamelCase  )       ->   Optional[Any]:
							lowerCAmelCase_      			=			"UNwant\u00E9d,running"
							lowerCAmelCase_      			=			"unwanted, running"
							return input_text, output_text




				def   __a		(      self  )       ->   Dict:
							lowerCAmelCase_      			=			self.tokenizer_class(self.vocab_file  )

							lowerCAmelCase_      			=			tokenizer.tokenize("UNwant\u00E9d,running"  )
							self.assertListEqual(_UpperCamelCase       ,       ["un", "##want", "##ed", ",", "runn", "##ing"]  )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase  )       ,       [9, 6, 7, 12, 10, 11]  )




				def   __a		(      self  )       ->   Any:
							if not self.test_rust_tokenizer:
										return

							lowerCAmelCase_      			=			self.get_tokenizer()
							lowerCAmelCase_      			=			self.get_rust_tokenizer()

							lowerCAmelCase_      			=			"UNwant\u00E9d,running"

							lowerCAmelCase_      			=			tokenizer.tokenize(_UpperCamelCase  )
							lowerCAmelCase_      			=			rust_tokenizer.tokenize(_UpperCamelCase  )
							self.assertListEqual(_UpperCamelCase       ,       _UpperCamelCase  )

							lowerCAmelCase_      			=			tokenizer.encode(_UpperCamelCase       ,       add_special_tokens=_UpperCamelCase  )
							lowerCAmelCase_      			=			rust_tokenizer.encode(_UpperCamelCase       ,       add_special_tokens=_UpperCamelCase  )
							self.assertListEqual(_UpperCamelCase       ,       _UpperCamelCase  )

							lowerCAmelCase_      			=			self.get_rust_tokenizer()
							lowerCAmelCase_      			=			tokenizer.encode(_UpperCamelCase  )
							lowerCAmelCase_      			=			rust_tokenizer.encode(_UpperCamelCase  )
							self.assertListEqual(_UpperCamelCase       ,       _UpperCamelCase  )

							# With lower casing
							lowerCAmelCase_      			=			self.get_tokenizer(do_lower_case=_UpperCamelCase  )
							lowerCAmelCase_      			=			self.get_rust_tokenizer(do_lower_case=_UpperCamelCase  )

							lowerCAmelCase_      			=			"UNwant\u00E9d,running"

							lowerCAmelCase_      			=			tokenizer.tokenize(_UpperCamelCase  )
							lowerCAmelCase_      			=			rust_tokenizer.tokenize(_UpperCamelCase  )
							self.assertListEqual(_UpperCamelCase       ,       _UpperCamelCase  )

							lowerCAmelCase_      			=			tokenizer.encode(_UpperCamelCase       ,       add_special_tokens=_UpperCamelCase  )
							lowerCAmelCase_      			=			rust_tokenizer.encode(_UpperCamelCase       ,       add_special_tokens=_UpperCamelCase  )
							self.assertListEqual(_UpperCamelCase       ,       _UpperCamelCase  )

							lowerCAmelCase_      			=			self.get_rust_tokenizer()
							lowerCAmelCase_      			=			tokenizer.encode(_UpperCamelCase  )
							lowerCAmelCase_      			=			rust_tokenizer.encode(_UpperCamelCase  )
							self.assertListEqual(_UpperCamelCase       ,       _UpperCamelCase  )




				def   __a		(      self  )       ->   str:
							lowerCAmelCase_      			=			BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"  )       ,       ["ah", "\u535A", "\u63A8", "zz"]  )




				def   __a		(      self  )       ->   Any:
							lowerCAmelCase_      			=			BasicTokenizer(do_lower_case=_UpperCamelCase  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "  )       ,       ["hello", "!", "how", "are", "you", "?"]  )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )       ,       ["hello"]  )




				def   __a		(      self  )       ->   str:
							lowerCAmelCase_      			=			BasicTokenizer(do_lower_case=_UpperCamelCase       ,       strip_accents=_UpperCamelCase  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )       ,       ["hällo", "!", "how", "are", "you", "?"]  )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )       ,       ["h\u00E9llo"]  )




				def   __a		(      self  )       ->   str:
							lowerCAmelCase_      			=			BasicTokenizer(do_lower_case=_UpperCamelCase       ,       strip_accents=_UpperCamelCase  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )       ,       ["hallo", "!", "how", "are", "you", "?"]  )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )       ,       ["hello"]  )




				def   __a		(      self  )       ->   Optional[Any]:
							lowerCAmelCase_      			=			BasicTokenizer(do_lower_case=_UpperCamelCase  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )       ,       ["hallo", "!", "how", "are", "you", "?"]  )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )       ,       ["hello"]  )




				def   __a		(      self  )       ->   Optional[Any]:
							lowerCAmelCase_      			=			BasicTokenizer(do_lower_case=_UpperCamelCase  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "  )       ,       ["HeLLo", "!", "how", "Are", "yoU", "?"]  )




				def   __a		(      self  )       ->   int:
							lowerCAmelCase_      			=			BasicTokenizer(do_lower_case=_UpperCamelCase       ,       strip_accents=_UpperCamelCase  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )       ,       ["HäLLo", "!", "how", "Are", "yoU", "?"]  )




				def   __a		(      self  )       ->   Dict:
							lowerCAmelCase_      			=			BasicTokenizer(do_lower_case=_UpperCamelCase       ,       strip_accents=_UpperCamelCase  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )       ,       ["HaLLo", "!", "how", "Are", "yoU", "?"]  )




				def   __a		(      self  )       ->   Dict:
							lowerCAmelCase_      			=			BasicTokenizer(do_lower_case=_UpperCamelCase       ,       never_split=["[UNK]"]  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"  )       ,       ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]  )




				def   __a		(      self  )       ->   Dict:
							lowerCAmelCase_      			=			BasicTokenizer()
							lowerCAmelCase_      			=			"a\n'll !!to?'d of, can't."
							lowerCAmelCase_      			=			["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
							self.assertListEqual(tokenizer.tokenize(_UpperCamelCase  )       ,       _UpperCamelCase  )




				def   __a		(      self  )       ->   Optional[int]:
							lowerCAmelCase_      			=			["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

							lowerCAmelCase_      			=			{}
							for i, token in enumerate(_UpperCamelCase  ):
										lowerCAmelCase_      			=			i
							lowerCAmelCase_      			=			WordpieceTokenizer(vocab=_UpperCamelCase       ,       unk_token="[UNK]"  )

							self.assertListEqual(tokenizer.tokenize(""  )       ,       []  )

							self.assertListEqual(tokenizer.tokenize("unwanted running"  )       ,       ["un", "##want", "##ed", "runn", "##ing"]  )

							self.assertListEqual(tokenizer.tokenize("unwantedX running"  )       ,       ["[UNK]", "runn", "##ing"]  )




				def   __a		(      self  )       ->   Optional[int]:
							self.assertTrue(_is_whitespace(" "  )  )
							self.assertTrue(_is_whitespace("\t"  )  )
							self.assertTrue(_is_whitespace("\r"  )  )
							self.assertTrue(_is_whitespace("\n"  )  )
							self.assertTrue(_is_whitespace("\u00A0"  )  )

							self.assertFalse(_is_whitespace("A"  )  )
							self.assertFalse(_is_whitespace("-"  )  )




				def   __a		(      self  )       ->   Optional[int]:
							self.assertTrue(_is_control("\u0005"  )  )

							self.assertFalse(_is_control("A"  )  )
							self.assertFalse(_is_control(" "  )  )
							self.assertFalse(_is_control("\t"  )  )
							self.assertFalse(_is_control("\r"  )  )




				def   __a		(      self  )       ->   List[str]:
							self.assertTrue(_is_punctuation("-"  )  )
							self.assertTrue(_is_punctuation("$"  )  )
							self.assertTrue(_is_punctuation("`"  )  )
							self.assertTrue(_is_punctuation("."  )  )

							self.assertFalse(_is_punctuation("A"  )  )
							self.assertFalse(_is_punctuation(" "  )  )




				def   __a		(      self  )       ->   Dict:
							lowerCAmelCase_      			=			self.get_tokenizer()
							lowerCAmelCase_      			=			self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(_UpperCamelCase  ) for t in ["Test", "\xad", "test"]]       ,       [["[UNK]"], [], ["[UNK]"]]  )

							self.assertListEqual(
							    [rust_tokenizer.tokenize(_UpperCamelCase  ) for t in ["Test", "\xad", "test"]]       ,       [["[UNK]"], [], ["[UNK]"]]  )




				@slow
				def   __a		(      self  )       ->   Union[str, Any]:
							lowerCAmelCase_      			=			self.tokenizer_class.from_pretrained("bert-base-uncased"  )

							lowerCAmelCase_      			=			tokenizer.encode("sequence builders"       ,       add_special_tokens=_UpperCamelCase  )
							lowerCAmelCase_      			=			tokenizer.encode("multi-sequence build"       ,       add_special_tokens=_UpperCamelCase  )

							lowerCAmelCase_      			=			tokenizer.build_inputs_with_special_tokens(_UpperCamelCase  )
							lowerCAmelCase_      			=			tokenizer.build_inputs_with_special_tokens(_UpperCamelCase       ,       _UpperCamelCase  )

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]




				def   __a		(      self  )       ->   int:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
													lowerCAmelCase_      			=			self.rust_tokenizer_class.from_pretrained(_UpperCamelCase       ,       **_UpperCamelCase  )

													lowerCAmelCase_      			=			f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
													lowerCAmelCase_      			=			tokenizer_r.encode_plus(
													    _UpperCamelCase       ,       return_attention_mask=_UpperCamelCase       ,       return_token_type_ids=_UpperCamelCase       ,       return_offsets_mapping=_UpperCamelCase       ,       add_special_tokens=_UpperCamelCase       ,       )

													lowerCAmelCase_      			=			tokenizer_r.do_lower_case if hasattr(_UpperCamelCase       ,       "do_lower_case"  ) else False
													lowerCAmelCase_      			=			(
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "A"),
													        ((1, 2), ","),
													        ((3, 5), "na"),
													        ((5, 6), "##ï"),
													        ((6, 8), "##ve"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "Allen"),
													        ((21, 23), "##NL"),
													        ((23, 24), "##P"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "a"),
													        ((1, 2), ","),
													        ((3, 8), "naive"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "allen"),
													        ((21, 23), "##nl"),
													        ((23, 24), "##p"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]       ,       tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]  )  )
													self.assertEqual([e[0] for e in expected_results]       ,       tokens["offset_mapping"]  )




				def   __a		(      self  )       ->   Any:
							lowerCAmelCase_      			=			["的", "人", "有"]
							lowerCAmelCase_      			=			"".join(_UpperCamelCase  )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
													lowerCAmelCase_      			=			True
													lowerCAmelCase_      			=			self.tokenizer_class.from_pretrained(_UpperCamelCase       ,       **_UpperCamelCase  )
													lowerCAmelCase_      			=			self.rust_tokenizer_class.from_pretrained(_UpperCamelCase       ,       **_UpperCamelCase  )

													lowerCAmelCase_      			=			tokenizer_p.encode(_UpperCamelCase       ,       add_special_tokens=_UpperCamelCase  )
													lowerCAmelCase_      			=			tokenizer_r.encode(_UpperCamelCase       ,       add_special_tokens=_UpperCamelCase  )

													lowerCAmelCase_      			=			tokenizer_r.convert_ids_to_tokens(_UpperCamelCase  )
													lowerCAmelCase_      			=			tokenizer_p.convert_ids_to_tokens(_UpperCamelCase  )

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(_UpperCamelCase       ,       _UpperCamelCase  )
													self.assertListEqual(_UpperCamelCase       ,       _UpperCamelCase  )

													lowerCAmelCase_      			=			False
													lowerCAmelCase_      			=			self.rust_tokenizer_class.from_pretrained(_UpperCamelCase       ,       **_UpperCamelCase  )
													lowerCAmelCase_      			=			self.tokenizer_class.from_pretrained(_UpperCamelCase       ,       **_UpperCamelCase  )

													lowerCAmelCase_      			=			tokenizer_r.encode(_UpperCamelCase       ,       add_special_tokens=_UpperCamelCase  )
													lowerCAmelCase_      			=			tokenizer_p.encode(_UpperCamelCase       ,       add_special_tokens=_UpperCamelCase  )

													lowerCAmelCase_      			=			tokenizer_r.convert_ids_to_tokens(_UpperCamelCase  )
													lowerCAmelCase_      			=			tokenizer_p.convert_ids_to_tokens(_UpperCamelCase  )

													# it is expected that only the first Chinese character is not preceded by "##".
													lowerCAmelCase_      			=			[
													    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(_UpperCamelCase  )
													]
													self.assertListEqual(_UpperCamelCase       ,       _UpperCamelCase  )
													self.assertListEqual(_UpperCamelCase       ,       _UpperCamelCase  )





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_A     =  logging.get_logger(__name__)

_A     =  {
    "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/config.json",
    "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/config.json",
    "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/config.json",
    "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/config.json",
    "bert-base-multilingual-uncased": "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/config.json",
    "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/config.json",
    "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/config.json",
    "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/config.json",
    "bert-large-uncased-whole-word-masking": (
        "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/config.json"
    ),
    "bert-large-cased-whole-word-masking": (
        "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/config.json"
    ),
    "bert-large-uncased-whole-word-masking-finetuned-squad": (
        "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/config.json"
    ),
    "bert-large-cased-whole-word-masking-finetuned-squad": (
        "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/config.json"
    ),
    "bert-base-cased-finetuned-mrpc": "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/config.json",
    "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/config.json",
    "bert-base-german-dbmdz-uncased": "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/config.json",
    "cl-tohoku/bert-base-japanese": "https://huggingface.co/cl-tohoku/bert-base-japanese/resolve/main/config.json",
    "cl-tohoku/bert-base-japanese-whole-word-masking": (
        "https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking/resolve/main/config.json"
    ),
    "cl-tohoku/bert-base-japanese-char": (
        "https://huggingface.co/cl-tohoku/bert-base-japanese-char/resolve/main/config.json"
    ),
    "cl-tohoku/bert-base-japanese-char-whole-word-masking": (
        "https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking/resolve/main/config.json"
    ),
    "TurkuNLP/bert-base-finnish-cased-v1": (
        "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/config.json"
    ),
    "TurkuNLP/bert-base-finnish-uncased-v1": (
        "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/config.json"
    ),
    "wietsedv/bert-base-dutch-cased": "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/config.json",
    # See all BERT models at https://huggingface.co/models?filter=bert
}





class 			_lowerCAmelCase      (			__a ):
				_lowercase    	='''bert'''




				def __init__(      self       ,       _UpperCamelCase=30_522       ,       _UpperCamelCase=768       ,       _UpperCamelCase=12       ,       _UpperCamelCase=12       ,       _UpperCamelCase=3_072       ,       _UpperCamelCase="gelu"       ,       _UpperCamelCase=0.1       ,       _UpperCamelCase=0.1       ,       _UpperCamelCase=512       ,       _UpperCamelCase=2       ,       _UpperCamelCase=0.02       ,       _UpperCamelCase=1e-1_2       ,       _UpperCamelCase=0       ,       _UpperCamelCase="absolute"       ,       _UpperCamelCase=True       ,       _UpperCamelCase=None       ,       **_UpperCamelCase       ,       )       ->   str:
							super().__init__(pad_token_id=_UpperCamelCase       ,       **_UpperCamelCase  )

							lowerCAmelCase_      			=			vocab_size
							lowerCAmelCase_      			=			hidden_size
							lowerCAmelCase_      			=			num_hidden_layers
							lowerCAmelCase_      			=			num_attention_heads
							lowerCAmelCase_      			=			hidden_act
							lowerCAmelCase_      			=			intermediate_size
							lowerCAmelCase_      			=			hidden_dropout_prob
							lowerCAmelCase_      			=			attention_probs_dropout_prob
							lowerCAmelCase_      			=			max_position_embeddings
							lowerCAmelCase_      			=			type_vocab_size
							lowerCAmelCase_      			=			initializer_range
							lowerCAmelCase_      			=			layer_norm_eps
							lowerCAmelCase_      			=			position_embedding_type
							lowerCAmelCase_      			=			use_cache
							lowerCAmelCase_      			=			classifier_dropout





class 			_lowerCAmelCase      (			__a ):




				@property
				def   __a		(      self  )       ->   Mapping[str, Mapping[int, str]]:
							if self.task == "multiple-choice":
										lowerCAmelCase_      			=			{0: "batch", 1: "choice", 2: "sequence"}
							else:
										lowerCAmelCase_      			=			{0: "batch", 1: "sequence"}
							return OrderedDict(
							    [
							        ("input_ids", dynamic_axis),
							        ("attention_mask", dynamic_axis),
							        ("token_type_ids", dynamic_axis),
							    ]  )



