






import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin
class 		SCREAMING_SNAKE_CASE__	(  UpperCAmelCase					,      unittest.TestCase     ):

  '''simple docstring'''







  _lowerCamelCase    =  TransfoXLTokenizer
  _lowerCamelCase    =  False
  _lowerCamelCase    =  False


  def  lowerCamelCase       (      self		:			Any							)						->	Tuple:


         """simple docstring"""
         super().setUp()

         _UpperCAmelCase							=    [
             """<unk>""",
             """[CLS]""",
             """[SEP]""",
             """want""",
             """unwanted""",
             """wa""",
             """un""",
             """running""",
             """,""",
             """low""",
             """l""",
         ]
         _UpperCAmelCase							=    os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES["""vocab_file"""]							)
         with open(self.vocab_file       ,				"""w"""       ,				encoding="""utf-8"""							) as vocab_writer:
                vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)


  def  lowerCamelCase       (      self		:			Dict       ,				**lowerCamelCase		:			Union[str, Any]							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    True
         return TransfoXLTokenizer.from_pretrained(self.tmpdirname       ,				**lowerCamelCase							)


  def  lowerCamelCase       (      self		:			List[str]       ,				lowerCamelCase		:			Union[str, Any]							)						->	Optional[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    """<unk> UNwanted , running"""
         _UpperCAmelCase							=    """<unk> unwanted, running"""
         return input_text, output_text


  def  lowerCamelCase       (      self		:			List[Any]							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    TransfoXLTokenizer(vocab_file=self.vocab_file       ,				lower_case=lowerCamelCase							)

         _UpperCAmelCase							=    tokenizer.tokenize("""<unk> UNwanted , running"""							)
         self.assertListEqual(lowerCamelCase       ,				["""<unk>""", """unwanted""", """,""", """running"""]							)

         self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase							)       ,				[0, 4, 8, 7]							)


  def  lowerCamelCase       (      self		:			Union[str, Any]							)						->	Optional[int]:


         """simple docstring"""
         _UpperCAmelCase							=    TransfoXLTokenizer(lower_case=lowerCamelCase							)

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """							)       ,				["""hello""", """!""", """how""", """are""", """you""", """?"""]							)


  def  lowerCamelCase       (      self		:			str							)						->	Union[str, Any]:


         """simple docstring"""
         _UpperCAmelCase							=    TransfoXLTokenizer(lower_case=lowerCamelCase							)

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """							)       ,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)


  def  lowerCamelCase       (      self		:			Tuple							)						->	List[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    TransfoXLTokenizer(lower_case=lowerCamelCase							)
         _UpperCAmelCase							=    """Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
         _UpperCAmelCase							=    [
             """Hello""",
             """(""",
             """bracket""",
             """)""",
             """and""",
             """side""",
             """@-@""",
             """scrolled""",
             """[""",
             """and""",
             """]""",
             """Henry""",
             """'s""",
             """$""",
             """5""",
             """@,@""",
             """000""",
             """with""",
             """3""",
             """@.@""",
             """34""",
             """m""",
             """.""",
             """What""",
             """'s""",
             """up""",
             """!""",
             """?""",
         ]

         self.assertListEqual(tokenizer.tokenize(lowerCamelCase							)       ,				lowerCamelCase							)

         self.assertEqual(tokenizer.convert_tokens_to_string(lowerCamelCase							)       ,				lowerCamelCase							)





  def  lowerCamelCase       (      self		:			Dict							)						->	Any:


         """simple docstring"""
         _UpperCAmelCase							=    self.get_tokenizer()
         _UpperCAmelCase							=    len(lowerCamelCase							)

         tokenizer.add_tokens(["""new1""", """new2"""]							)
         tokenizer.move_added_token("""new1"""       ,				1							)

         # Check that moved token is not copied (duplicate)
         self.assertEqual(len(lowerCamelCase							)       ,				original_len + 2							)
         # Check that token is moved to specified id
         self.assertEqual(tokenizer.encode("""new1"""							)       ,				[1]							)
         self.assertEqual(tokenizer.decode([1]							)       ,				"""new1"""							)







import os
import sys
import unittest


__a:     Tuple   					=					os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, '''utils'''))

import check_dummies  # noqa: E402
from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init  # noqa: E402


# Align TRANSFORMERS_PATH in check_dummies with the current path
__a:     Tuple   					=					os.path.join(git_repo_path, '''src''', '''transformers''')

__a:     Optional[Any]   					=					'''
{0} = None
'''

__a:     List[Any]   					=					'''
class {0}(metaclass=DummyObject):
    _backends = {1}

    def __init__(self, *args, **kwargs):
        requires_backends(self, {1})
'''


__a:     int   					=					'''
def {0}(*args, **kwargs):
    requires_backends({0}, {1})
'''
class 		SCREAMING_SNAKE_CASE__	(  unittest.TestCase     ):

  '''simple docstring'''







  def  lowerCamelCase       (      self		:			Tuple							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    find_backend("""    _import_structure[\"models.albert\"].append(\"AlbertTokenizerFast\")"""							)
         self.assertIsNone(lowerCamelCase							)

         _UpperCAmelCase							=    find_backend("""    if not is_tokenizers_available():"""							)
         self.assertEqual(lowerCamelCase       ,				"""tokenizers"""							)

         _UpperCAmelCase							=    find_backend("""    if not is_tensorflow_text_available():"""							)
         self.assertEqual(lowerCamelCase       ,				"""tensorflow_text"""							)

         _UpperCAmelCase							=    find_backend("""    if not (is_sentencepiece_available() and is_tokenizers_available()):"""							)
         self.assertEqual(lowerCamelCase       ,				"""sentencepiece_and_tokenizers"""							)

         _UpperCAmelCase							=    find_backend(
             """    if not (is_sentencepiece_available() and is_tensorflow_text_available()):"""							)
         self.assertEqual(lowerCamelCase       ,				"""sentencepiece_and_tensorflow_text"""							)

         _UpperCAmelCase							=    find_backend(
             """    if not (is_sentencepiece_available() and is_tokenizers_available() and is_vision_available()):"""							)
         self.assertEqual(lowerCamelCase       ,				"""sentencepiece_and_tokenizers_and_vision"""							)


  def  lowerCamelCase       (      self		:			int							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    read_init()
         # We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects
         self.assertIn("""torch"""       ,				lowerCamelCase							)
         self.assertIn("""tensorflow_text"""       ,				lowerCamelCase							)
         self.assertIn("""sentencepiece_and_tokenizers"""       ,				lowerCamelCase							)

         # Likewise, we can't assert on the exact content of a key
         self.assertIn("""BertModel"""       ,				objects["""torch"""]							)
         self.assertIn("""TFBertModel"""       ,				objects["""tf"""]							)
         self.assertIn("""FlaxBertModel"""       ,				objects["""flax"""]							)
         self.assertIn("""BertModel"""       ,				objects["""torch"""]							)
         self.assertIn("""TFBertTokenizer"""       ,				objects["""tensorflow_text"""]							)
         self.assertIn("""convert_slow_tokenizer"""       ,				objects["""sentencepiece_and_tokenizers"""]							)


  def  lowerCamelCase       (      self		:			int							)						->	Any:


         """simple docstring"""
         _UpperCAmelCase							=    create_dummy_object("""CONSTANT"""       ,				"""'torch'"""							)
         self.assertEqual(lowerCamelCase       ,				"""\nCONSTANT = None\n"""							)

         _UpperCAmelCase							=    create_dummy_object("""function"""       ,				"""'torch'"""							)
         self.assertEqual(
             lowerCamelCase       ,				"""\ndef function(*args, **kwargs):\n    requires_backends(function, 'torch')\n"""							)

         _UpperCAmelCase							=    """
class FakeClass(metaclass=DummyObject):
    _backends = 'torch'

    def __init__(self, *args, **kwargs):
        requires_backends(self, 'torch')
"""
         _UpperCAmelCase							=    create_dummy_object("""FakeClass"""       ,				"""'torch'"""							)
         self.assertEqual(lowerCamelCase       ,				lowerCamelCase							)





  def  lowerCamelCase       (      self		:			Dict							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    """# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..utils import DummyObject, requires_backends


CONSTANT = None


def function(*args, **kwargs):
    requires_backends(function, [\"torch\"])


class FakeClass(metaclass=DummyObject):
    _backends = [\"torch\"]

    def __init__(self, *args, **kwargs):
        requires_backends(self, [\"torch\"])
"""
         _UpperCAmelCase							=    create_dummy_files({"""torch""": ["""CONSTANT""", """function""", """FakeClass"""]}							)
         self.assertEqual(dummy_files["""torch"""]       ,				lowerCamelCase							)