

import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class SCREAMING_SNAKE_CASE	(  a_   ,							unittest.TestCase ):
  """simple docstring"""






  lowerCamelCase    :			List[Any]						      =RealmTokenizer
  lowerCamelCase    :			Union[str, Any]						      =RealmTokenizerFast
  lowerCamelCase    :			Any						      =True
  lowerCamelCase    :			Dict						      =True
  lowerCamelCase    :			Union[str, Any]						      =filter_non_english







  def        SCREAMING_SNAKE_CASE    (			self :    Optional[Any]    )							->	Tuple:






     """simple docstring"""







     super().setUp()

     __lowerCAmelCase  :			Tuple					    =						[
         """[UNK]""",
         """[CLS]""",
         """[SEP]""",
         """[PAD]""",
         """[MASK]""",
         """want""",
         """##want""",
         """##ed""",
         """wa""",
         """un""",
         """runn""",
         """##ing""",
         """,""",
         """low""",
         """lowest""",
     ]
     __lowerCAmelCase  :			int					    =						os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["""vocab_file"""]    )
     with open(self.vocab_file       ,							"""w"""       ,							encoding="""utf-8"""    ) as vocab_writer:
        vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )







  def        SCREAMING_SNAKE_CASE    (			self :    Tuple       ,							lowerCAmelCase :    Optional[Any]    )							->	Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :			Union[str, Any]					    =						"""UNwant\u00E9d,running"""
     __lowerCAmelCase  :			List[Any]					    =						"""unwanted, running"""
     return input_text, output_text







  def        SCREAMING_SNAKE_CASE    (			self :    List[Any]    )							->	Any:






     """simple docstring"""







     __lowerCAmelCase  :			List[str]					    =						self.tokenizer_class(self.vocab_file    )

     __lowerCAmelCase  :			Any					    =						tokenizer.tokenize("""UNwant\u00E9d,running"""    )
     self.assertListEqual(snake_case__       ,							["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]    )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__    )       ,							[9, 6, 7, 12, 10, 11]    )







  def        SCREAMING_SNAKE_CASE    (			self :    List[str]    )							->	List[Any]:






     """simple docstring"""







     if not self.test_rust_tokenizer:
        return

     __lowerCAmelCase  :			Any					    =						self.get_tokenizer()
     __lowerCAmelCase  :			Union[str, Any]					    =						self.get_rust_tokenizer()

     __lowerCAmelCase  :			str					    =						"""UNwant\u00E9d,running"""

     __lowerCAmelCase  :			Optional[Any]					    =						tokenizer.tokenize(snake_case__    )
     __lowerCAmelCase  :			Dict					    =						rust_tokenizer.tokenize(snake_case__    )
     self.assertListEqual(snake_case__       ,							snake_case__    )

     __lowerCAmelCase  :			Any					    =						tokenizer.encode(snake_case__       ,							add_special_tokens=snake_case__    )
     __lowerCAmelCase  :			str					    =						rust_tokenizer.encode(snake_case__       ,							add_special_tokens=snake_case__    )
     self.assertListEqual(snake_case__       ,							snake_case__    )

     __lowerCAmelCase  :			Any					    =						self.get_rust_tokenizer()
     __lowerCAmelCase  :			Dict					    =						tokenizer.encode(snake_case__    )
     __lowerCAmelCase  :			Optional[int]					    =						rust_tokenizer.encode(snake_case__    )
     self.assertListEqual(snake_case__       ,							snake_case__    )

     # With lower casing
     __lowerCAmelCase  :			int					    =						self.get_tokenizer(do_lower_case=snake_case__    )
     __lowerCAmelCase  :			str					    =						self.get_rust_tokenizer(do_lower_case=snake_case__    )

     __lowerCAmelCase  :			Dict					    =						"""UNwant\u00E9d,running"""

     __lowerCAmelCase  :			Dict					    =						tokenizer.tokenize(snake_case__    )
     __lowerCAmelCase  :			Union[str, Any]					    =						rust_tokenizer.tokenize(snake_case__    )
     self.assertListEqual(snake_case__       ,							snake_case__    )

     __lowerCAmelCase  :			Optional[int]					    =						tokenizer.encode(snake_case__       ,							add_special_tokens=snake_case__    )
     __lowerCAmelCase  :			int					    =						rust_tokenizer.encode(snake_case__       ,							add_special_tokens=snake_case__    )
     self.assertListEqual(snake_case__       ,							snake_case__    )

     __lowerCAmelCase  :			Union[str, Any]					    =						self.get_rust_tokenizer()
     __lowerCAmelCase  :			Dict					    =						tokenizer.encode(snake_case__    )
     __lowerCAmelCase  :			Optional[Any]					    =						rust_tokenizer.encode(snake_case__    )
     self.assertListEqual(snake_case__       ,							snake_case__    )







  def        SCREAMING_SNAKE_CASE    (			self :    List[Any]    )							->	int:






     """simple docstring"""







     __lowerCAmelCase  :			List[str]					    =						BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""    )       ,							["""ah""", """\u535A""", """\u63A8""", """zz"""]    )







  def        SCREAMING_SNAKE_CASE    (			self :    int    )							->	int:






     """simple docstring"""







     __lowerCAmelCase  :			Union[str, Any]					    =						BasicTokenizer(do_lower_case=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )       ,							["""hello""", """!""", """how""", """are""", """you""", """?"""]    )
     self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )       ,							["""hello"""]    )







  def        SCREAMING_SNAKE_CASE    (			self :    Optional[Any]    )							->	Tuple:






     """simple docstring"""







     __lowerCAmelCase  :			List[str]					    =						BasicTokenizer(do_lower_case=snake_case__       ,							strip_accents=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )       ,							["""hällo""", """!""", """how""", """are""", """you""", """?"""]    )
     self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )       ,							["""h\u00E9llo"""]    )







  def        SCREAMING_SNAKE_CASE    (			self :    str    )							->	Any:






     """simple docstring"""







     __lowerCAmelCase  :			str					    =						BasicTokenizer(do_lower_case=snake_case__       ,							strip_accents=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )       ,							["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
     self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )       ,							["""hello"""]    )







  def        SCREAMING_SNAKE_CASE    (			self :    List[str]    )							->	Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :			Tuple					    =						BasicTokenizer(do_lower_case=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )       ,							["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
     self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )       ,							["""hello"""]    )







  def        SCREAMING_SNAKE_CASE    (			self :    Optional[int]    )							->	Dict:






     """simple docstring"""







     __lowerCAmelCase  :			Tuple					    =						BasicTokenizer(do_lower_case=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )       ,							["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )







  def        SCREAMING_SNAKE_CASE    (			self :    Optional[Any]    )							->	Dict:






     """simple docstring"""







     __lowerCAmelCase  :			List[str]					    =						BasicTokenizer(do_lower_case=snake_case__       ,							strip_accents=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )       ,							["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )







  def        SCREAMING_SNAKE_CASE    (			self :    List[str]    )							->	str:






     """simple docstring"""







     __lowerCAmelCase  :			Union[str, Any]					    =						BasicTokenizer(do_lower_case=snake_case__       ,							strip_accents=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )       ,							["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )







  def        SCREAMING_SNAKE_CASE    (			self :    List[str]    )							->	Dict:






     """simple docstring"""







     __lowerCAmelCase  :			List[str]					    =						BasicTokenizer(do_lower_case=snake_case__       ,							never_split=["""[UNK]"""]    )

     self.assertListEqual(
         tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""    )       ,							["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]    )







  def        SCREAMING_SNAKE_CASE    (			self :    int    )							->	List[Any]:






     """simple docstring"""







     __lowerCAmelCase  :			int					    =						["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

     __lowerCAmelCase  :			Dict					    =						{}
     for i, token in enumerate(snake_case__    ):
        __lowerCAmelCase  :			str					    =						i
     __lowerCAmelCase  :			Any					    =						WordpieceTokenizer(vocab=snake_case__       ,							unk_token="""[UNK]"""    )

     self.assertListEqual(tokenizer.tokenize(""""""    )       ,							[]    )

     self.assertListEqual(tokenizer.tokenize("""unwanted running"""    )       ,							["""un""", """##want""", """##ed""", """runn""", """##ing"""]    )

     self.assertListEqual(tokenizer.tokenize("""unwantedX running"""    )       ,							["""[UNK]""", """runn""", """##ing"""]    )







  def        SCREAMING_SNAKE_CASE    (			self :    Dict    )							->	Union[str, Any]:






     """simple docstring"""







     self.assertTrue(_is_whitespace(""" """    )    )
     self.assertTrue(_is_whitespace("""\t"""    )    )
     self.assertTrue(_is_whitespace("""\r"""    )    )
     self.assertTrue(_is_whitespace("""\n"""    )    )
     self.assertTrue(_is_whitespace("""\u00A0"""    )    )

     self.assertFalse(_is_whitespace("""A"""    )    )
     self.assertFalse(_is_whitespace("""-"""    )    )







  def        SCREAMING_SNAKE_CASE    (			self :    int    )							->	List[Any]:






     """simple docstring"""







     self.assertTrue(_is_control("""\u0005"""    )    )

     self.assertFalse(_is_control("""A"""    )    )
     self.assertFalse(_is_control(""" """    )    )
     self.assertFalse(_is_control("""\t"""    )    )
     self.assertFalse(_is_control("""\r"""    )    )







  def        SCREAMING_SNAKE_CASE    (			self :    str    )							->	List[str]:






     """simple docstring"""







     self.assertTrue(_is_punctuation("""-"""    )    )
     self.assertTrue(_is_punctuation("""$"""    )    )
     self.assertTrue(_is_punctuation("""`"""    )    )
     self.assertTrue(_is_punctuation("""."""    )    )

     self.assertFalse(_is_punctuation("""A"""    )    )
     self.assertFalse(_is_punctuation(""" """    )    )







  def        SCREAMING_SNAKE_CASE    (			self :    Dict    )							->	Union[str, Any]:






     """simple docstring"""







     __lowerCAmelCase  :			List[Any]					    =						self.get_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(snake_case__    ) for t in ["""Test""", """\xad""", """test"""]]       ,							[["""[UNK]"""], [], ["""[UNK]"""]]    )

     if self.test_rust_tokenizer:
        __lowerCAmelCase  :			int					    =						self.get_rust_tokenizer()
        self.assertListEqual(
            [rust_tokenizer.tokenize(snake_case__    ) for t in ["""Test""", """\xad""", """test"""]]       ,							[["""[UNK]"""], [], ["""[UNK]"""]]    )







  @slow
  def        SCREAMING_SNAKE_CASE    (			self :    Any    )							->	Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :			Any					    =						self.tokenizer_class.from_pretrained("""bert-base-uncased"""    )

     __lowerCAmelCase  :			Union[str, Any]					    =						tokenizer.encode("""sequence builders"""       ,							add_special_tokens=snake_case__    )
     __lowerCAmelCase  :			int					    =						tokenizer.encode("""multi-sequence build"""       ,							add_special_tokens=snake_case__    )

     __lowerCAmelCase  :			str					    =						tokenizer.build_inputs_with_special_tokens(snake_case__    )
     __lowerCAmelCase  :			str					    =						tokenizer.build_inputs_with_special_tokens(snake_case__       ,							snake_case__    )

     assert encoded_sentence == [1_01] + text + [1_02]
     assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]







  def        SCREAMING_SNAKE_CASE    (			self :    Any    )							->	Optional[Any]:






     """simple docstring"""







     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
           __lowerCAmelCase  :			List[Any]					    =						self.rust_tokenizer_class.from_pretrained(snake_case__       ,							**snake_case__    )

           __lowerCAmelCase  :			Tuple					    =						f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
           __lowerCAmelCase  :			List[str]					    =						tokenizer_r.encode_plus(
               snake_case__       ,							return_attention_mask=snake_case__       ,							return_token_type_ids=snake_case__       ,							return_offsets_mapping=snake_case__       ,							add_special_tokens=snake_case__       ,							)

           __lowerCAmelCase  :			Optional[int]					    =						tokenizer_r.do_lower_case if hasattr(snake_case__       ,							"""do_lower_case"""    ) else False
           __lowerCAmelCase  :			Union[str, Any]					    =						(
               [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), """A"""),
                   ((1, 2), ""","""),
                   ((3, 5), """na"""),
                   ((5, 6), """##ï"""),
                   ((6, 8), """##ve"""),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), """Allen"""),
                   ((21, 23), """##NL"""),
                   ((23, 24), """##P"""),
                   ((25, 33), """sentence"""),
                   ((33, 34), """."""),
                   ((0, 0), tokenizer_r.sep_token),
               ]
               if not do_lower_case
               else [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), """a"""),
                   ((1, 2), ""","""),
                   ((3, 8), """naive"""),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), """allen"""),
                   ((21, 23), """##nl"""),
                   ((23, 24), """##p"""),
                   ((25, 33), """sentence"""),
                   ((33, 34), """."""),
                   ((0, 0), tokenizer_r.sep_token),
               ]
           )

           self.assertEqual(
               [e[1] for e in expected_results]       ,							tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]    )    )
           self.assertEqual([e[0] for e in expected_results]       ,							tokens["""offset_mapping"""]    )







  @slow
  def        SCREAMING_SNAKE_CASE    (			self :    Dict    )							->	int:






     """simple docstring"""







     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
           __lowerCAmelCase  :			Tuple					    =						self.rust_tokenizer_class.from_pretrained(snake_case__       ,							**snake_case__    )
           __lowerCAmelCase  :			Any					    =						self.tokenizer_class.from_pretrained(snake_case__       ,							**snake_case__    )
           __lowerCAmelCase  :			Union[str, Any]					    =						[["""Hello world!""", """Nice to meet you!"""], ["""The cute cat.""", """The adorable dog."""]]

           __lowerCAmelCase  :			Union[str, Any]					    =						tokenizer_r.batch_encode_candidates(snake_case__       ,							max_length=10       ,							return_tensors="""np"""    )
           __lowerCAmelCase  :			str					    =						tokenizer_p.batch_encode_candidates(snake_case__       ,							max_length=10       ,							return_tensors="""np"""    )

           __lowerCAmelCase  :			Optional[Any]					    =						(2, 2, 10)

           self.assertEqual(encoded_sentence_r["""input_ids"""].shape       ,							snake_case__    )
           self.assertEqual(encoded_sentence_r["""attention_mask"""].shape       ,							snake_case__    )
           self.assertEqual(encoded_sentence_r["""token_type_ids"""].shape       ,							snake_case__    )

           self.assertEqual(encoded_sentence_p["""input_ids"""].shape       ,							snake_case__    )
           self.assertEqual(encoded_sentence_p["""attention_mask"""].shape       ,							snake_case__    )
           self.assertEqual(encoded_sentence_p["""token_type_ids"""].shape       ,							snake_case__    )










"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase_		:			str				   =					logging.get_logger(__name__)

lowerCAmelCase_		:			Optional[int]				   =					{
    '''google/reformer-crime-and-punishment''': (
        '''https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/config.json'''
    ),
    '''google/reformer-enwik8''': '''https://huggingface.co/google/reformer-enwik8/resolve/main/config.json''',
}





class      UpperCamelCase_       (				a_     ):
  _A				:    int =		'reformer'
  _A				:    Any =		['past_buckets_states']
  _A				:    Optional[Any] =		{}



  def __init__(					self       ,						snake_case__=64       ,						snake_case__=["local", "lsh", "local", "lsh", "local", "lsh"]       ,						snake_case__=1.0       ,						snake_case__=True       ,						snake_case__=[64, 64]       ,						snake_case__=[64, 1_92]       ,						snake_case__=0       ,						snake_case__=2       ,						snake_case__=5_12       ,						snake_case__=None       ,						snake_case__="relu"       ,						snake_case__=0.05       ,						snake_case__=2_56       ,						snake_case__=0.02       ,						snake_case__=False       ,						snake_case__=1e-12       ,						snake_case__=1       ,						snake_case__=0       ,						snake_case__=0.05       ,						snake_case__=64       ,						snake_case__=64       ,						snake_case__=0.0       ,						snake_case__=1       ,						snake_case__=0       ,						snake_case__=40_96       ,						snake_case__=12       ,						snake_case__=None       ,						snake_case__=1       ,						snake_case__=0       ,						snake_case__=3_20       ,						snake_case__=False       ,						snake_case__=True       ,						snake_case__=None       ,						**snake_case__       ,						)	->	List[Any]:



         """simple docstring"""





         UpperCAmelCase					=	hash_seed
         UpperCAmelCase					=	vocab_size
         UpperCAmelCase					=	attention_head_size
         UpperCAmelCase					=	hidden_size
         UpperCAmelCase					=	num_attention_heads
         UpperCAmelCase					=	num_hashes
         UpperCAmelCase					=	len(snake_case__ )
         UpperCAmelCase					=	tuple(snake_case__ ) if isinstance(snake_case__       ,						snake_case__ ) else num_buckets
         UpperCAmelCase					=	lsh_attn_chunk_length
         UpperCAmelCase					=	local_attn_chunk_length
         UpperCAmelCase					=	lsh_num_chunks_after
         UpperCAmelCase					=	lsh_num_chunks_before
         UpperCAmelCase					=	local_num_chunks_after
         UpperCAmelCase					=	local_num_chunks_before
         UpperCAmelCase					=	hidden_act
         UpperCAmelCase					=	feed_forward_size
         UpperCAmelCase					=	hidden_dropout_prob
         UpperCAmelCase					=	lsh_attention_probs_dropout_prob
         UpperCAmelCase					=	local_attention_probs_dropout_prob
         UpperCAmelCase					=	max_position_embeddings
         UpperCAmelCase					=	initializer_range
         UpperCAmelCase					=	layer_norm_eps
         UpperCAmelCase					=	axial_pos_embds
         UpperCAmelCase					=	tuple(snake_case__ )
         UpperCAmelCase					=	tuple(snake_case__ )
         UpperCAmelCase					=	axial_norm_std
         UpperCAmelCase					=	chunk_size_lm_head
         UpperCAmelCase					=	attn_layers
         UpperCAmelCase					=	use_cache
         UpperCAmelCase					=	classifier_dropout
         super().__init__(
             pad_token_id=snake_case__       ,						eos_token_id=snake_case__       ,						is_decoder=snake_case__       ,						tie_word_embeddings=snake_case__       ,						**snake_case__       ,						)
