





'''simple docstring'''






import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 		lowerCamelCase	(   _A   , unittest.TestCase							):
	snake_case_   = BertTokenizer
	snake_case_   = BertTokenizerFast
	snake_case_   = True
	snake_case_   = True
	snake_case_   = filter_non_english



	def 	_lowerCamelCase     ( self						):
			super().setUp()

			lowerCAmelCase  :     Union[str, Any]       						= [
			    "[UNK]",
			    "[CLS]",
			    "[SEP]",
			    "[PAD]",
			    "[MASK]",
			    "want",
			    "##want",
			    "##ed",
			    "wa",
			    "un",
			    "runn",
			    "##ing",
			    ",",
			    "low",
			    "lowest",
			]
			lowerCAmelCase  :     Optional[int]       						= os.path.join(self.tmpdirname		,   VOCAB_FILES_NAMES["vocab_file"]						)
			with open(self.vocab_file		,   "w"		,   encoding="utf-8"						) as vocab_writer:
					vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)



	def 	_lowerCamelCase     ( self		,   a_						):
			lowerCAmelCase  :     Tuple       						= "UNwant\u00E9d,running"
			lowerCAmelCase  :     int       						= "unwanted, running"
			return input_text, output_text



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Optional[Any]       						= self.tokenizer_class(self.vocab_file						)

			lowerCAmelCase  :     Optional[int]       						= tokenizer.tokenize("UNwant\u00E9d,running"						)
			self.assertListEqual(a_		,   ["un", "##want", "##ed", ",", "runn", "##ing"]						)
			self.assertListEqual(tokenizer.convert_tokens_to_ids(a_						)		,   [9, 6, 7, 12, 10, 11]						)



	def 	_lowerCamelCase     ( self						):
			if not self.test_rust_tokenizer:
					return

			lowerCAmelCase  :     Union[str, Any]       						= self.get_tokenizer()
			lowerCAmelCase  :     Union[str, Any]       						= self.get_rust_tokenizer()

			lowerCAmelCase  :     Tuple       						= "UNwant\u00E9d,running"

			lowerCAmelCase  :     Any       						= tokenizer.tokenize(a_						)
			lowerCAmelCase  :     int       						= rust_tokenizer.tokenize(a_						)
			self.assertListEqual(a_		,   a_						)

			lowerCAmelCase  :     Optional[int]       						= tokenizer.encode(a_		,   add_special_tokens=a_						)
			lowerCAmelCase  :     Dict       						= rust_tokenizer.encode(a_		,   add_special_tokens=a_						)
			self.assertListEqual(a_		,   a_						)

			lowerCAmelCase  :     Any       						= self.get_rust_tokenizer()
			lowerCAmelCase  :     Optional[Any]       						= tokenizer.encode(a_						)
			lowerCAmelCase  :     List[Any]       						= rust_tokenizer.encode(a_						)
			self.assertListEqual(a_		,   a_						)

			# With lower casing
			lowerCAmelCase  :     Any       						= self.get_tokenizer(do_lower_case=a_						)
			lowerCAmelCase  :     int       						= self.get_rust_tokenizer(do_lower_case=a_						)

			lowerCAmelCase  :     List[str]       						= "UNwant\u00E9d,running"

			lowerCAmelCase  :     str       						= tokenizer.tokenize(a_						)
			lowerCAmelCase  :     List[Any]       						= rust_tokenizer.tokenize(a_						)
			self.assertListEqual(a_		,   a_						)

			lowerCAmelCase  :     Union[str, Any]       						= tokenizer.encode(a_		,   add_special_tokens=a_						)
			lowerCAmelCase  :     Tuple       						= rust_tokenizer.encode(a_		,   add_special_tokens=a_						)
			self.assertListEqual(a_		,   a_						)

			lowerCAmelCase  :     Tuple       						= self.get_rust_tokenizer()
			lowerCAmelCase  :     str       						= tokenizer.encode(a_						)
			lowerCAmelCase  :     List[Any]       						= rust_tokenizer.encode(a_						)
			self.assertListEqual(a_		,   a_						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     int       						= BasicTokenizer()

			self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"						)		,   ["ah", "\u535A", "\u63A8", "zz"]						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     int       						= BasicTokenizer(do_lower_case=a_						)

			self.assertListEqual(
			    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)		,   ["hello", "!", "how", "are", "you", "?"]						)
			self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)		,   ["hello"]						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Dict       						= BasicTokenizer(do_lower_case=a_		,   strip_accents=a_						)

			self.assertListEqual(
			    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)		,   ["hällo", "!", "how", "are", "you", "?"]						)
			self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)		,   ["h\u00E9llo"]						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Optional[Any]       						= BasicTokenizer(do_lower_case=a_		,   strip_accents=a_						)

			self.assertListEqual(
			    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)		,   ["hallo", "!", "how", "are", "you", "?"]						)
			self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)		,   ["hello"]						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Dict       						= BasicTokenizer(do_lower_case=a_						)

			self.assertListEqual(
			    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)		,   ["hallo", "!", "how", "are", "you", "?"]						)
			self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)		,   ["hello"]						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Tuple       						= BasicTokenizer(do_lower_case=a_						)

			self.assertListEqual(
			    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)		,   ["HeLLo", "!", "how", "Are", "yoU", "?"]						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     int       						= BasicTokenizer(do_lower_case=a_		,   strip_accents=a_						)

			self.assertListEqual(
			    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)		,   ["HäLLo", "!", "how", "Are", "yoU", "?"]						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Any       						= BasicTokenizer(do_lower_case=a_		,   strip_accents=a_						)

			self.assertListEqual(
			    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)		,   ["HaLLo", "!", "how", "Are", "yoU", "?"]						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Any       						= BasicTokenizer(do_lower_case=a_		,   never_split=["[UNK]"]						)

			self.assertListEqual(
			    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"						)		,   ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     str       						= BasicTokenizer()
			lowerCAmelCase  :     Optional[Any]       						= "a\n'll !!to?'d of, can't."
			lowerCAmelCase  :     Union[str, Any]       						= ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
			self.assertListEqual(tokenizer.tokenize(a_						)		,   a_						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Optional[Any]       						= ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

			lowerCAmelCase  :     Optional[Any]       						= {}
			for i, token in enumerate(a_						):
					lowerCAmelCase  :     List[str]       						= i
			lowerCAmelCase  :     List[Any]       						= WordpieceTokenizer(vocab=a_		,   unk_token="[UNK]"						)

			self.assertListEqual(tokenizer.tokenize(""						)		,   []						)

			self.assertListEqual(tokenizer.tokenize("unwanted running"						)		,   ["un", "##want", "##ed", "runn", "##ing"]						)

			self.assertListEqual(tokenizer.tokenize("unwantedX running"						)		,   ["[UNK]", "runn", "##ing"]						)



	def 	_lowerCamelCase     ( self						):
			self.assertTrue(_is_whitespace(" "						)						)
			self.assertTrue(_is_whitespace("\t"						)						)
			self.assertTrue(_is_whitespace("\r"						)						)
			self.assertTrue(_is_whitespace("\n"						)						)
			self.assertTrue(_is_whitespace("\u00A0"						)						)

			self.assertFalse(_is_whitespace("A"						)						)
			self.assertFalse(_is_whitespace("-"						)						)



	def 	_lowerCamelCase     ( self						):
			self.assertTrue(_is_control("\u0005"						)						)

			self.assertFalse(_is_control("A"						)						)
			self.assertFalse(_is_control(" "						)						)
			self.assertFalse(_is_control("\t"						)						)
			self.assertFalse(_is_control("\r"						)						)



	def 	_lowerCamelCase     ( self						):
			self.assertTrue(_is_punctuation("-"						)						)
			self.assertTrue(_is_punctuation("$"						)						)
			self.assertTrue(_is_punctuation("`"						)						)
			self.assertTrue(_is_punctuation("."						)						)

			self.assertFalse(_is_punctuation("A"						)						)
			self.assertFalse(_is_punctuation(" "						)						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Dict       						= self.get_tokenizer()
			lowerCAmelCase  :     Union[str, Any]       						= self.get_rust_tokenizer()

			# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
			self.assertListEqual([tokenizer.tokenize(a_						) for t in ["Test", "\xad", "test"]]		,   [["[UNK]"], [], ["[UNK]"]]						)

			self.assertListEqual(
			    [rust_tokenizer.tokenize(a_						) for t in ["Test", "\xad", "test"]]		,   [["[UNK]"], [], ["[UNK]"]]						)



	@slow
	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Union[str, Any]       						= self.tokenizer_class.from_pretrained("bert-base-uncased"						)

			lowerCAmelCase  :     str       						= tokenizer.encode("sequence builders"		,   add_special_tokens=a_						)
			lowerCAmelCase  :     int       						= tokenizer.encode("multi-sequence build"		,   add_special_tokens=a_						)

			lowerCAmelCase  :     int       						= tokenizer.build_inputs_with_special_tokens(a_						)
			lowerCAmelCase  :     List[Any]       						= tokenizer.build_inputs_with_special_tokens(a_		,   a_						)

			assert encoded_sentence == [101] + text + [102]
			assert encoded_pair == [101] + text + [102] + text_a + [102]



	def 	_lowerCamelCase     ( self						):
			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
							lowerCAmelCase  :     Tuple       						= self.rust_tokenizer_class.from_pretrained(a_		,   **a_						)

							lowerCAmelCase  :     List[Any]       						= F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
							lowerCAmelCase  :     str       						= tokenizer_r.encode_plus(
							    a_		,   return_attention_mask=a_		,   return_token_type_ids=a_		,   return_offsets_mapping=a_		,   add_special_tokens=a_		,   )

							lowerCAmelCase  :     List[str]       						= tokenizer_r.do_lower_case if hasattr(a_		,   "do_lower_case"						) else False
							lowerCAmelCase  :     Optional[Any]       						= (
							    [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), "A"),
							        ((1, 2), ","),
							        ((3, 5), "na"),
							        ((5, 6), "##ï"),
							        ((6, 8), "##ve"),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), "Allen"),
							        ((21, 23), "##NL"),
							        ((23, 24), "##P"),
							        ((25, 33), "sentence"),
							        ((33, 34), "."),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							    if not do_lower_case
							    else [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), "a"),
							        ((1, 2), ","),
							        ((3, 8), "naive"),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), "allen"),
							        ((21, 23), "##nl"),
							        ((23, 24), "##p"),
							        ((25, 33), "sentence"),
							        ((33, 34), "."),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							)

							self.assertEqual(
							    [e[1] for e in expected_results]		,   tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]						)						)
							self.assertEqual([e[0] for e in expected_results]		,   tokens["offset_mapping"]						)





	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Optional[int]       						= ["的", "人", "有"]
			lowerCAmelCase  :     Optional[Any]       						= "".join(a_						)
			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
							lowerCAmelCase  :     List[Any]       						= True
							lowerCAmelCase  :     int       						= self.tokenizer_class.from_pretrained(a_		,   **a_						)
							lowerCAmelCase  :     List[str]       						= self.rust_tokenizer_class.from_pretrained(a_		,   **a_						)

							lowerCAmelCase  :     Tuple       						= tokenizer_p.encode(a_		,   add_special_tokens=a_						)
							lowerCAmelCase  :     Any       						= tokenizer_r.encode(a_		,   add_special_tokens=a_						)

							lowerCAmelCase  :     Union[str, Any]       						= tokenizer_r.convert_ids_to_tokens(a_						)
							lowerCAmelCase  :     int       						= tokenizer_p.convert_ids_to_tokens(a_						)

							# it is expected that each Chinese character is not preceded by "##"
							self.assertListEqual(a_		,   a_						)
							self.assertListEqual(a_		,   a_						)

							lowerCAmelCase  :     List[str]       						= False
							lowerCAmelCase  :     List[Any]       						= self.rust_tokenizer_class.from_pretrained(a_		,   **a_						)
							lowerCAmelCase  :     Dict       						= self.tokenizer_class.from_pretrained(a_		,   **a_						)

							lowerCAmelCase  :     Tuple       						= tokenizer_r.encode(a_		,   add_special_tokens=a_						)
							lowerCAmelCase  :     List[str]       						= tokenizer_p.encode(a_		,   add_special_tokens=a_						)

							lowerCAmelCase  :     Union[str, Any]       						= tokenizer_r.convert_ids_to_tokens(a_						)
							lowerCAmelCase  :     Dict       						= tokenizer_p.convert_ids_to_tokens(a_						)

							# it is expected that only the first Chinese character is not preceded by "##".
							lowerCAmelCase  :     Tuple       						= [
							    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(a_						)
							]
							self.assertListEqual(a_		,   a_						)
							self.assertListEqual(a_		,   a_						)










'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import _LazyModule


lowerCAmelCase									=   {"""tokenization_wav2vec2_phoneme""": ["""Wav2Vec2PhonemeCTCTokenizer"""]}


if TYPE_CHECKING:
					from .tokenization_wavaveca_phoneme import WavaVecaPhonemeCTCTokenizer
else:
					import sys

					lowerCAmelCase									=   _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



