





import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 	UpperCamelCase_						(     _lowerCamelCase							,     unittest.TestCase    ):
					lowerCAmelCase_							=		RealmTokenizer
					lowerCAmelCase_							=		RealmTokenizerFast
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		filter_non_english




					def 		lowerCAmelCase	(						self     )						->							Optional[int]:
										super().setUp()

										_snake_case													= [
										    '[UNK]',
										    '[CLS]',
										    '[SEP]',
										    '[PAD]',
										    '[MASK]',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										_snake_case													= os.path.join(self.tmpdirname ,  VOCAB_FILES_NAMES['vocab_file']     )
										with open(self.vocab_file ,  'w' ,  encoding='utf-8'     ) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )




					def 		lowerCAmelCase	(						self ,  lowerCAmelCase_     )						->							int:
										_snake_case													= 'UNwant\u00E9d,running'
										_snake_case													= 'unwanted, running'
										return input_text, output_text




					def 		lowerCAmelCase	(						self     )						->							Optional[Any]:
										_snake_case													= self.tokenizer_class(self.vocab_file     )

										_snake_case													= tokenizer.tokenize('UNwant\u00E9d,running'     )
										self.assertListEqual(_lowercase ,  ['un', '##want', '##ed', ',', 'runn', '##ing']     )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase     ) ,  [9, 6, 7, 12, 10, 11]     )




					def 		lowerCAmelCase	(						self     )						->							Optional[int]:
										if not self.test_rust_tokenizer:
															return

										_snake_case													= self.get_tokenizer()
										_snake_case													= self.get_rust_tokenizer()

										_snake_case													= 'UNwant\u00E9d,running'

										_snake_case													= tokenizer.tokenize(_lowercase     )
										_snake_case													= rust_tokenizer.tokenize(_lowercase     )
										self.assertListEqual(_lowercase ,  _lowercase     )

										_snake_case													= tokenizer.encode(_lowercase ,  add_special_tokens=_lowercase     )
										_snake_case													= rust_tokenizer.encode(_lowercase ,  add_special_tokens=_lowercase     )
										self.assertListEqual(_lowercase ,  _lowercase     )

										_snake_case													= self.get_rust_tokenizer()
										_snake_case													= tokenizer.encode(_lowercase     )
										_snake_case													= rust_tokenizer.encode(_lowercase     )
										self.assertListEqual(_lowercase ,  _lowercase     )

										# With lower casing
										_snake_case													= self.get_tokenizer(do_lower_case=_lowercase     )
										_snake_case													= self.get_rust_tokenizer(do_lower_case=_lowercase     )

										_snake_case													= 'UNwant\u00E9d,running'

										_snake_case													= tokenizer.tokenize(_lowercase     )
										_snake_case													= rust_tokenizer.tokenize(_lowercase     )
										self.assertListEqual(_lowercase ,  _lowercase     )

										_snake_case													= tokenizer.encode(_lowercase ,  add_special_tokens=_lowercase     )
										_snake_case													= rust_tokenizer.encode(_lowercase ,  add_special_tokens=_lowercase     )
										self.assertListEqual(_lowercase ,  _lowercase     )

										_snake_case													= self.get_rust_tokenizer()
										_snake_case													= tokenizer.encode(_lowercase     )
										_snake_case													= rust_tokenizer.encode(_lowercase     )
										self.assertListEqual(_lowercase ,  _lowercase     )




					def 		lowerCAmelCase	(						self     )						->							Tuple:
										_snake_case													= BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'     ) ,  ['ah', '\u535A', '\u63A8', 'zz']     )




					def 		lowerCAmelCase	(						self     )						->							Dict:
										_snake_case													= BasicTokenizer(do_lower_case=_lowercase     )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '     ) ,  ['hello', '!', 'how', 'are', 'you', '?']     )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     ) ,  ['hello']     )




					def 		lowerCAmelCase	(						self     )						->							Optional[Any]:
										_snake_case													= BasicTokenizer(do_lower_case=_lowercase ,  strip_accents=_lowercase     )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     ) ,  ['hällo', '!', 'how', 'are', 'you', '?']     )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     ) ,  ['h\u00E9llo']     )




					def 		lowerCAmelCase	(						self     )						->							Dict:
										_snake_case													= BasicTokenizer(do_lower_case=_lowercase ,  strip_accents=_lowercase     )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     ) ,  ['hallo', '!', 'how', 'are', 'you', '?']     )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     ) ,  ['hello']     )




					def 		lowerCAmelCase	(						self     )						->							Any:
										_snake_case													= BasicTokenizer(do_lower_case=_lowercase     )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     ) ,  ['hallo', '!', 'how', 'are', 'you', '?']     )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     ) ,  ['hello']     )




					def 		lowerCAmelCase	(						self     )						->							Tuple:
										_snake_case													= BasicTokenizer(do_lower_case=_lowercase     )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '     ) ,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?']     )




					def 		lowerCAmelCase	(						self     )						->							int:
										_snake_case													= BasicTokenizer(do_lower_case=_lowercase ,  strip_accents=_lowercase     )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     ) ,  ['HäLLo', '!', 'how', 'Are', 'yoU', '?']     )




					def 		lowerCAmelCase	(						self     )						->							Dict:
										_snake_case													= BasicTokenizer(do_lower_case=_lowercase ,  strip_accents=_lowercase     )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     ) ,  ['HaLLo', '!', 'how', 'Are', 'yoU', '?']     )




					def 		lowerCAmelCase	(						self     )						->							str:
										_snake_case													= BasicTokenizer(do_lower_case=_lowercase ,  never_split=['[UNK]']     )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'     ) ,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']     )




					def 		lowerCAmelCase	(						self     )						->							Any:
										_snake_case													= ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

										_snake_case													= {}
										for i, token in enumerate(_lowercase     ):
															_snake_case													= i
										_snake_case													= WordpieceTokenizer(vocab=_lowercase ,  unk_token='[UNK]'     )

										self.assertListEqual(tokenizer.tokenize(''     ) ,  []     )

										self.assertListEqual(tokenizer.tokenize('unwanted running'     ) ,  ['un', '##want', '##ed', 'runn', '##ing']     )

										self.assertListEqual(tokenizer.tokenize('unwantedX running'     ) ,  ['[UNK]', 'runn', '##ing']     )




					def 		lowerCAmelCase	(						self     )						->							Optional[Any]:
										self.assertTrue(_is_whitespace(' '     )     )
										self.assertTrue(_is_whitespace('\t'     )     )
										self.assertTrue(_is_whitespace('\r'     )     )
										self.assertTrue(_is_whitespace('\n'     )     )
										self.assertTrue(_is_whitespace('\u00A0'     )     )

										self.assertFalse(_is_whitespace('A'     )     )
										self.assertFalse(_is_whitespace('-'     )     )




					def 		lowerCAmelCase	(						self     )						->							Any:
										self.assertTrue(_is_control('\u0005'     )     )

										self.assertFalse(_is_control('A'     )     )
										self.assertFalse(_is_control(' '     )     )
										self.assertFalse(_is_control('\t'     )     )
										self.assertFalse(_is_control('\r'     )     )




					def 		lowerCAmelCase	(						self     )						->							Union[str, Any]:
										self.assertTrue(_is_punctuation('-'     )     )
										self.assertTrue(_is_punctuation('$'     )     )
										self.assertTrue(_is_punctuation('`'     )     )
										self.assertTrue(_is_punctuation('.'     )     )

										self.assertFalse(_is_punctuation('A'     )     )
										self.assertFalse(_is_punctuation(' '     )     )




					def 		lowerCAmelCase	(						self     )						->							Union[str, Any]:
										_snake_case													= self.get_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(_lowercase     ) for t in ['Test', '\xad', 'test']] ,  [['[UNK]'], [], ['[UNK]']]     )

										if self.test_rust_tokenizer:
															_snake_case													= self.get_rust_tokenizer()
															self.assertListEqual(
															    [rust_tokenizer.tokenize(_lowercase     ) for t in ['Test', '\xad', 'test']] ,  [['[UNK]'], [], ['[UNK]']]     )




					@slow
					def 		lowerCAmelCase	(						self     )						->							List[Any]:
										_snake_case													= self.tokenizer_class.from_pretrained('bert-base-uncased'     )

										_snake_case													= tokenizer.encode('sequence builders' ,  add_special_tokens=_lowercase     )
										_snake_case													= tokenizer.encode('multi-sequence build' ,  add_special_tokens=_lowercase     )

										_snake_case													= tokenizer.build_inputs_with_special_tokens(_lowercase     )
										_snake_case													= tokenizer.build_inputs_with_special_tokens(_lowercase ,  _lowercase     )

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]




					def 		lowerCAmelCase	(						self     )						->							Optional[Any]:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
																				_snake_case													= self.rust_tokenizer_class.from_pretrained(_lowercase ,  **_lowercase     )

																				_snake_case													= F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																				_snake_case													= tokenizer_r.encode_plus(
																				    _lowercase ,  return_attention_mask=_lowercase ,  return_token_type_ids=_lowercase ,  return_offsets_mapping=_lowercase ,  add_special_tokens=_lowercase ,  )

																				_snake_case													= tokenizer_r.do_lower_case if hasattr(_lowercase ,  'do_lower_case'     ) else False
																				_snake_case													= (
																				    [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'A'),
																				        ((1, 2), ','),
																				        ((3, 5), 'na'),
																				        ((5, 6), '##ï'),
																				        ((6, 8), '##ve'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'Allen'),
																				        ((21, 23), '##NL'),
																				        ((23, 24), '##P'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				    if not do_lower_case
																				    else [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'a'),
																				        ((1, 2), ','),
																				        ((3, 8), 'naive'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'allen'),
																				        ((21, 23), '##nl'),
																				        ((23, 24), '##p'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				)

																				self.assertEqual(
																				    [e[1] for e in expected_results] ,  tokenizer_r.convert_ids_to_tokens(tokens['input_ids']     )     )
																				self.assertEqual([e[0] for e in expected_results] ,  tokens['offset_mapping']     )






					@slow
					def 		lowerCAmelCase	(						self     )						->							Optional[int]:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
																				_snake_case													= self.rust_tokenizer_class.from_pretrained(_lowercase ,  **_lowercase     )
																				_snake_case													= self.tokenizer_class.from_pretrained(_lowercase ,  **_lowercase     )
																				_snake_case													= [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

																				_snake_case													= tokenizer_r.batch_encode_candidates(_lowercase ,  max_length=10 ,  return_tensors='np'     )
																				_snake_case													= tokenizer_p.batch_encode_candidates(_lowercase ,  max_length=10 ,  return_tensors='np'     )

																				_snake_case													= (2, 2, 10)

																				self.assertEqual(encoded_sentence_r['input_ids'].shape ,  _lowercase     )
																				self.assertEqual(encoded_sentence_r['attention_mask'].shape ,  _lowercase     )
																				self.assertEqual(encoded_sentence_r['token_type_ids'].shape ,  _lowercase     )

																				self.assertEqual(encoded_sentence_p['input_ids'].shape ,  _lowercase     )
																				self.assertEqual(encoded_sentence_p['attention_mask'].shape ,  _lowercase     )
																				self.assertEqual(encoded_sentence_p['token_type_ids'].shape ,  _lowercase     )






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


SCREAMING_SNAKE_CASE						:int      =			{"""configuration_yoso""": ["""YOSO_PRETRAINED_CONFIG_ARCHIVE_MAP""", """YosoConfig"""]}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 SCREAMING_SNAKE_CASE						:List[Any]      =			[
     """YOSO_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """YosoForMaskedLM""",
     """YosoForMultipleChoice""",
     """YosoForQuestionAnswering""",
     """YosoForSequenceClassification""",
     """YosoForTokenClassification""",
     """YosoLayer""",
     """YosoModel""",
     """YosoPreTrainedModel""",
 ]


if TYPE_CHECKING:
 from .configuration_yoso import YOSO_PRETRAINED_CONFIG_ARCHIVE_MAP, YosoConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_yoso import (
      YOSO_PRETRAINED_MODEL_ARCHIVE_LIST,
      YosoForMaskedLM,
      YosoForMultipleChoice,
      YosoForQuestionAnswering,
      YosoForSequenceClassification,
      YosoForTokenClassification,
      YosoLayer,
      YosoModel,
      YosoPreTrainedModel,
  )


else:
 import sys

 SCREAMING_SNAKE_CASE						:str      =			_LazyModule(__name__, globals()["""__file__"""], _import_structure)
