

import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class SCREAMING_SNAKE_CASE	(  a_   ,							unittest.TestCase ):
  """simple docstring"""






  lowerCamelCase    :			List[Any]						      =PreTrainedTokenizerFast
  lowerCamelCase    :			Tuple						      =False
  lowerCamelCase    :			Optional[int]						      =True
  lowerCamelCase    :			Union[str, Any]						      ="tokenizer_file"







  def        SCREAMING_SNAKE_CASE    (			self :    int    )							->	Tuple:






     """simple docstring"""







     __lowerCAmelCase  :			Union[str, Any]					    =						False  # because we don't have pretrained_vocab_files_map
     super().setUp()
     __lowerCAmelCase  :			Union[str, Any]					    =						True

     __lowerCAmelCase  :			List[str]					    =						["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
     __lowerCAmelCase  :			int					    =						"""SaulLu/dummy-tokenizer-bytelevel-bpe"""

     # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
     __lowerCAmelCase  :			int					    =						[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

     __lowerCAmelCase  :			Optional[Any]					    =						PreTrainedTokenizerFast.from_pretrained(model_paths[0]    )
     tokenizer.save_pretrained(self.tmpdirname    )







  def        SCREAMING_SNAKE_CASE    (			self :    Optional[int]    )							->	Union[str, Any]:






     """simple docstring"""







     pass







  def        SCREAMING_SNAKE_CASE    (			self :    Optional[Any]    )							->	Dict:






     """simple docstring"""







     pass







  def        SCREAMING_SNAKE_CASE    (			self :    Tuple    )							->	Optional[Any]:






     """simple docstring"""







     pass







  def        SCREAMING_SNAKE_CASE    (			self :    Dict    )							->	List[str]:






     """simple docstring"""







     pass







  def        SCREAMING_SNAKE_CASE    (			self :    Optional[int]    )							->	Dict:






     """simple docstring"""







     __lowerCAmelCase  :			Dict					    =						self.tmpdirname
     # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
           try:
              __lowerCAmelCase  :			str					    =						tempfile.mkdtemp()
              __lowerCAmelCase  :			Optional[Any]					    =						self.rust_tokenizer_class.from_pretrained(lowerCAmelCase       ,							**lowerCAmelCase    )

              tokenizer.save_pretrained(self.tmpdirname    )
              super().test_training_new_tokenizer()
           finally:
              # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
              # is restored
              shutil.rmtree(self.tmpdirname    )
              __lowerCAmelCase  :			List[str]					    =						tmpdirname_orig







  def        SCREAMING_SNAKE_CASE    (			self :    Optional[int]    )							->	Any:






     """simple docstring"""







     __lowerCAmelCase  :			str					    =						self.tmpdirname
     # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
           try:
              __lowerCAmelCase  :			Tuple					    =						tempfile.mkdtemp()
              __lowerCAmelCase  :			Dict					    =						self.rust_tokenizer_class.from_pretrained(lowerCAmelCase       ,							**lowerCAmelCase    )

              tokenizer.save_pretrained(self.tmpdirname    )
              super().test_training_new_tokenizer_with_special_tokens_change()
           finally:
              # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
              # is restored
              shutil.rmtree(self.tmpdirname    )
              __lowerCAmelCase  :			Union[str, Any]					    =						tmpdirname_orig







  def        SCREAMING_SNAKE_CASE    (			self :    str    )							->	Dict:






     """simple docstring"""







     __lowerCAmelCase  :			Optional[Any]					    =						self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name    )

     __lowerCAmelCase  :			Union[str, Any]					    =						("""a""" for _ in range(10_00    ))
     __lowerCAmelCase  :			int					    =						tokenizer.train_new_from_iterator(text_iterator=lowerCAmelCase       ,							length=10_00       ,							vocab_size=50    )

     __lowerCAmelCase  :			str					    =						new_tokenizer.encode("""a🤗"""    )
     self.assertEqual(lowerCAmelCase       ,							[64, 1_72, 2_53, 97, 2_45]    )






@require_tokenizers
class SCREAMING_SNAKE_CASE	(  unittest.TestCase ):
  """simple docstring"""






  def        SCREAMING_SNAKE_CASE    (			self :    Optional[Any]    )							->	List[str]:






     """simple docstring"""







     __lowerCAmelCase  :			Tuple					    =						AutoTokenizer.from_pretrained("""bert-base-cased"""    )
     __lowerCAmelCase  :			Optional[Any]					    =						json.loads(tokenizer._tokenizer.to_str()    )
     __lowerCAmelCase  :			Dict					    =						len(lowerCAmelCase    )

     with tempfile.TemporaryDirectory() as tmp_dir:
        # Hack to save this in the tokenizer_config.json
        __lowerCAmelCase  :			Optional[int]					    =						["""tokenizer.4.0.0.json"""]
        tokenizer.save_pretrained(lowerCAmelCase    )
        json.dump(lowerCAmelCase       ,							open(os.path.join(lowerCAmelCase       ,							"""tokenizer.4.0.0.json"""    )       ,							"""w"""    )    )

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        __lowerCAmelCase  :			Union[str, Any]					    =						AutoTokenizer.from_pretrained(lowerCAmelCase    )
        self.assertEqual(len(lowerCAmelCase    )       ,							len(lowerCAmelCase    ) + 1    )
        __lowerCAmelCase  :			List[Any]					    =						json.loads(new_tokenizer._tokenizer.to_str()    )
        self.assertIn("""huggingface"""       ,							json_tokenizer["""model"""]["""vocab"""]    )

        # Will need to be adjusted if we reach v42 and this test is still here.
        # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
        shutil.move(os.path.join(lowerCAmelCase       ,							"""tokenizer.4.0.0.json"""    )       ,							os.path.join(lowerCAmelCase       ,							"""tokenizer.42.0.0.json"""    )    )
        __lowerCAmelCase  :			Optional[int]					    =						["""tokenizer.42.0.0.json"""]
        tokenizer.save_pretrained(lowerCAmelCase    )
        __lowerCAmelCase  :			Optional[int]					    =						AutoTokenizer.from_pretrained(lowerCAmelCase    )
        self.assertEqual(len(lowerCAmelCase    )       ,							len(lowerCAmelCase    )    )
        __lowerCAmelCase  :			str					    =						json.loads(new_tokenizer._tokenizer.to_str()    )
        self.assertNotIn("""huggingface"""       ,							json_tokenizer["""model"""]["""vocab"""]    )







  def        SCREAMING_SNAKE_CASE    (			self :    Optional[Any]    )							->	List[Any]:






     """simple docstring"""







     __lowerCAmelCase  :			Optional[int]					    =						"""hf-internal-testing/test-two-tokenizers"""

     # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
     __lowerCAmelCase  :			List[Any]					    =						AutoTokenizer.from_pretrained(lowerCAmelCase    )
     self.assertEqual(len(lowerCAmelCase    )       ,							2_89_97    )
     __lowerCAmelCase  :			List[Any]					    =						json.loads(tokenizer._tokenizer.to_str()    )
     self.assertIn("""huggingface"""       ,							json_tokenizer["""model"""]["""vocab"""]    )

     # Testing an older version by monkey-patching the version in the module it's used.
     import transformers as old_transformers

     __lowerCAmelCase  :			Dict					    =						"""3.0.0"""
     __lowerCAmelCase  :			str					    =						old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCAmelCase    )
     self.assertEqual(len(lowerCAmelCase    )       ,							2_89_96    )
     __lowerCAmelCase  :			Any					    =						json.loads(old_tokenizer._tokenizer.to_str()    )
     self.assertNotIn("""huggingface"""       ,							json_tokenizer["""model"""]["""vocab"""]    )






@require_tokenizers
class SCREAMING_SNAKE_CASE	(  unittest.TestCase ):
  """simple docstring"""






  def        SCREAMING_SNAKE_CASE    (			self :    Dict    )							->	Tuple:






     """simple docstring"""







     __lowerCAmelCase  :			Optional[int]					    =						PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""    )
     __lowerCAmelCase  :			List[str]					    =						"""The Matrix is a 1999 science fiction action film."""

     with concurrent.futures.ThreadPoolExecutor() as executor:
        __lowerCAmelCase  :			Union[str, Any]					    =						[executor.submit(self.fetch       ,							lowerCAmelCase       ,							lowerCAmelCase    ) for i in range(10    )]
        __lowerCAmelCase  :			List[Any]					    =						[future.result() for future in futures]
        self.assertEqual(lowerCAmelCase       ,							[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10    )]    )







  def        SCREAMING_SNAKE_CASE    (			self :    List[Any]       ,							lowerCAmelCase :    List[str]       ,							lowerCAmelCase :    List[str]    )							->	Any:






     """simple docstring"""







     return tokenizer.encode(lowerCAmelCase       ,							truncation="""longest_first"""       ,							padding="""longest"""    )





import importlib.metadata
import subprocess
import sys





def 	snake_case_      (__A     :						bool = False     )			->	str:
   __lowerCAmelCase  :			Any					    =						False
   if "IPython" in sys.modules:
      __lowerCAmelCase  :			Optional[int]					    =						"""google.colab""" in str(sys.modules["""IPython"""].get_ipython()     )

   if in_colab:
      if upgrade:
         __lowerCAmelCase  :			int					    =						["""pip""", """install""", """-U""", """torch"""]
         subprocess.run(__A					,   check=__A     )
      # get the current version of torch
      __lowerCAmelCase  :			List[Any]					    =						importlib.metadata.version("""torch"""     )
      __lowerCAmelCase  :			List[Any]					    =						torch_version[: torch_version.rindex("""."""     )]
      __lowerCAmelCase  :			int					    =						f'''https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-{torch_version_trunc}-cp37-cp37m-linux_x86_64.whl'''
      __lowerCAmelCase  :			List[Any]					    =						["""pip""", """install""", xla_wheel]
      subprocess.run(__A					,   check=__A     )
   else:
      raise RuntimeError("""`install_xla` utility works only on google colab."""     )


