import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowercase_     :   str 				=				get_tests_dir('fixtures/test_sentencepiece.model')






@require_sentencepiece
@require_tokenizers
class 		_lowerCamelCase		(   UpperCamelCase_    ,     unittest.TestCase    ):
    __a  =      ReformerTokenizer
    __a  =      ReformerTokenizerFast
    __a  =      True
    __a  =      False
    __a  =      True
    def 						UpperCamelCase_				(  self   )       ->					List[Any]:
     super().setUp()

     SCREAMING_SNAKE_CASE__:   Any=      ReformerTokenizer(lowerCAmelCase , keep_accents=lowerCAmelCase   )
     tokenizer.save_pretrained(self.tmpdirname   )
    def 						UpperCamelCase_				(  self   )       ->					Optional[int]:
     SCREAMING_SNAKE_CASE__:   List[Any]=      '''<s>'''
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase   ) , lowerCAmelCase   )
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase   ) , lowerCAmelCase   )
    def 						UpperCamelCase_				(  self   )       ->					Union[str, Any]:
     SCREAMING_SNAKE_CASE__:   List[str]=      list(self.get_tokenizer().get_vocab().keys()   )

     self.assertEqual(vocab_keys[0] , '''<unk>'''   )
     self.assertEqual(vocab_keys[1] , '''<s>'''   )
     self.assertEqual(vocab_keys[-1] , '''j'''   )
     self.assertEqual(len(lowerCAmelCase   ) , 1000   )
    def 						UpperCamelCase_				(  self   )       ->					List[Any]:
     self.assertEqual(self.get_tokenizer().vocab_size , 1000   )
    def 						UpperCamelCase_				(  self   )       ->					Tuple:
     if not self.test_rust_tokenizer:
      return

     SCREAMING_SNAKE_CASE__:   Any=      self.get_tokenizer()
     SCREAMING_SNAKE_CASE__:   int=      self.get_rust_tokenizer()

     SCREAMING_SNAKE_CASE__:   List[Any]=      '''I was born in 92000, and this is falsé.'''

     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      tokenizer.tokenize(lowerCAmelCase   )
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      rust_tokenizer.tokenize(lowerCAmelCase   )
     self.assertListEqual(lowerCAmelCase , lowerCAmelCase   )

     SCREAMING_SNAKE_CASE__:   List[Any]=      tokenizer.encode(lowerCAmelCase , add_special_tokens=lowerCAmelCase   )
     SCREAMING_SNAKE_CASE__:   Dict=      rust_tokenizer.encode(lowerCAmelCase , add_special_tokens=lowerCAmelCase   )
     self.assertListEqual(lowerCAmelCase , lowerCAmelCase   )

     SCREAMING_SNAKE_CASE__:   List[str]=      self.get_rust_tokenizer()
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      tokenizer.encode(lowerCAmelCase   )
     SCREAMING_SNAKE_CASE__:   List[str]=      rust_tokenizer.encode(lowerCAmelCase   )
     self.assertListEqual(lowerCAmelCase , lowerCAmelCase   )
    def 						UpperCamelCase_				(  self , lowerCAmelCase=15   )       ->					Any:
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
       SCREAMING_SNAKE_CASE__:   str=      self.rust_tokenizer_class.from_pretrained(lowerCAmelCase , **lowerCAmelCase   )

       # Simple input
       SCREAMING_SNAKE_CASE__:   Optional[Any]=      '''This is a simple input'''
       SCREAMING_SNAKE_CASE__:   Optional[int]=      ['''This is a simple input 1''', '''This is a simple input 2''']
       SCREAMING_SNAKE_CASE__:   Optional[int]=      ('''This is a simple input''', '''This is a pair''')
       SCREAMING_SNAKE_CASE__:   int=      [
           ('''This is a simple input 1''', '''This is a simple input 2'''),
           ('''This is a simple pair 1''', '''This is a simple pair 2'''),
       ]

       # Simple input tests
       self.assertRaises(lowerCAmelCase , tokenizer_r.encode , lowerCAmelCase , max_length=lowerCAmelCase , padding='''max_length'''   )

       # Simple input
       self.assertRaises(lowerCAmelCase , tokenizer_r.encode_plus , lowerCAmelCase , max_length=lowerCAmelCase , padding='''max_length'''   )

       # Simple input
       self.assertRaises(
           lowerCAmelCase , tokenizer_r.batch_encode_plus , lowerCAmelCase , max_length=lowerCAmelCase , padding='''max_length''' , )

       # Pair input
       self.assertRaises(lowerCAmelCase , tokenizer_r.encode , lowerCAmelCase , max_length=lowerCAmelCase , padding='''max_length'''   )

       # Pair input
       self.assertRaises(lowerCAmelCase , tokenizer_r.encode_plus , lowerCAmelCase , max_length=lowerCAmelCase , padding='''max_length'''   )

       # Pair input
       self.assertRaises(
           lowerCAmelCase , tokenizer_r.batch_encode_plus , lowerCAmelCase , max_length=lowerCAmelCase , padding='''max_length''' , )
    def 						UpperCamelCase_				(  self   )       ->					str:
     pass
    def 						UpperCamelCase_				(  self   )       ->					List[str]:
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      ReformerTokenizer(lowerCAmelCase , keep_accents=lowerCAmelCase   )

     SCREAMING_SNAKE_CASE__:   Any=      tokenizer.tokenize('''This is a test'''   )
     self.assertListEqual(lowerCAmelCase , ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']   )

     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(lowerCAmelCase   ) , [285, 46, 10, 170, 382] , )

     SCREAMING_SNAKE_CASE__:   Optional[int]=      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''   )
     self.assertListEqual(
         lowerCAmelCase , [
             SPIECE_UNDERLINE + '''I''',
             SPIECE_UNDERLINE + '''was''',
             SPIECE_UNDERLINE + '''b''',
             '''or''',
             '''n''',
             SPIECE_UNDERLINE + '''in''',
             SPIECE_UNDERLINE + '''''',
             '''9''',
             '''2''',
             '''0''',
             '''0''',
             '''0''',
             ''',''',
             SPIECE_UNDERLINE + '''and''',
             SPIECE_UNDERLINE + '''this''',
             SPIECE_UNDERLINE + '''is''',
             SPIECE_UNDERLINE + '''f''',
             '''al''',
             '''s''',
             '''é''',
             '''.''',
         ] , )
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      tokenizer.convert_tokens_to_ids(lowerCAmelCase   )
     self.assertListEqual(
         lowerCAmelCase , [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4] , )

     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      tokenizer.convert_ids_to_tokens(lowerCAmelCase   )
     self.assertListEqual(
         lowerCAmelCase , [
             SPIECE_UNDERLINE + '''I''',
             SPIECE_UNDERLINE + '''was''',
             SPIECE_UNDERLINE + '''b''',
             '''or''',
             '''n''',
             SPIECE_UNDERLINE + '''in''',
             SPIECE_UNDERLINE + '''''',
             '''<unk>''',
             '''2''',
             '''0''',
             '''0''',
             '''0''',
             ''',''',
             SPIECE_UNDERLINE + '''and''',
             SPIECE_UNDERLINE + '''this''',
             SPIECE_UNDERLINE + '''is''',
             SPIECE_UNDERLINE + '''f''',
             '''al''',
             '''s''',
             '''<unk>''',
             '''.''',
         ] , )
    @cached_property
    def 						UpperCamelCase_				(  self   )       ->					Any:
     return ReformerTokenizer.from_pretrained('''google/reformer-crime-and-punishment'''   )
    @slow
    def 						UpperCamelCase_				(  self   )       ->					List[Any]:
     SCREAMING_SNAKE_CASE__:   Optional[int]=      '''Hello World!'''
     SCREAMING_SNAKE_CASE__:   List[str]=      [126, 32, 262, 152, 38, 72, 287]

     self.assertListEqual(lowerCAmelCase , self.big_tokenizer.encode(lowerCAmelCase   )   )
    @slow
    def 						UpperCamelCase_				(  self   )       ->					Optional[Any]:
     SCREAMING_SNAKE_CASE__:   Tuple=      (
         '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
         ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
     )
     SCREAMING_SNAKE_CASE__:   List[str]=      [
         108,
         265,
         24,
         111,
         4,
         258,
         156,
         35,
         28,
         275,
         3,
         259,
         297,
         260,
         84,
         4,
         35,
         110,
         44,
         8,
         259,
         91,
         268,
         21,
         11,
         209,
         274,
         109,
         266,
         277,
         117,
         86,
         93,
         315,
         258,
         278,
         258,
         277,
         258,
         0,
         258,
         288,
         258,
         319,
         258,
         0,
         258,
         0,
         258,
         0,
         258,
         0,
         258,
         287,
         258,
         315,
         258,
         289,
         258,
         278,
         99,
         269,
         266,
         262,
         8,
         259,
         241,
         4,
         217,
         230,
         268,
         266,
         55,
         168,
         106,
         75,
         193,
         266,
         223,
         27,
         49,
         26,
         282,
         25,
         264,
         299,
         19,
         26,
         0,
         258,
         277,
         117,
         86,
         93,
         176,
         183,
         270,
         11,
         262,
         42,
         61,
         265,
     ]

     self.assertListEqual(lowerCAmelCase , self.big_tokenizer.encode(lowerCAmelCase   )   )
    @require_torch
    @slow
    def 						UpperCamelCase_				(  self   )       ->					Optional[int]:
     import torch

     from transformers import ReformerConfig, ReformerModel

     # Build sequence
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      list(self.big_tokenizer.get_vocab().keys()   )[:10]
     SCREAMING_SNAKE_CASE__:   Optional[int]=      ''' '''.join(lowerCAmelCase   )
     SCREAMING_SNAKE_CASE__:   List[Any]=      self.big_tokenizer.encode_plus(lowerCAmelCase , return_tensors='''pt'''   )
     SCREAMING_SNAKE_CASE__:   int=      self.big_tokenizer.batch_encode_plus([sequence, sequence] , return_tensors='''pt'''   )

     SCREAMING_SNAKE_CASE__:   Dict=      ReformerConfig()
     # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
     SCREAMING_SNAKE_CASE__:   int=      encoded_sequence['''input_ids'''].shape
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      ReformerModel(lowerCAmelCase   )

     # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
     assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

     with torch.no_grad():
      model(**lowerCAmelCase   )
      model(**lowerCAmelCase   )



    @slow
    def 						UpperCamelCase_				(  self   )       ->					Tuple:
     # fmt: off
     SCREAMING_SNAKE_CASE__:   List[Any]=      {'''input_ids''': [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
     # fmt: on

     # This tokenizer does not know some characters like ")".
     # That is the reason why we use very simple texts here.
     # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      [
         '''This is a very simple sentence.''',
         '''The quick brown fox jumps over the lazy dog.''',
     ]

     self.tokenizer_integration_test_util(
         expected_encoding=lowerCAmelCase , model_name='''google/reformer-crime-and-punishment''' , revision='''0e6c3decb8211d49bf881013425dc8b0448b3f5a''' , padding=lowerCAmelCase , sequences=lowerCAmelCase , )


import re







def 			A__      (	snake_case_ :		str      ):

 if len(re.findall('''[ATCG]'''       ,				snake_case_      )      ) != len(snake_case_      ):
  raise ValueError('''Invalid Strand'''      )

 return dna.translate(dna.maketrans('''ATCG'''       ,				'''TAGC'''      )      )


if __name__ == "__main__":
      import doctest

      doctest.testmod()

