

"""simple docstring"""




import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class    __lowercase	(			__a		,	unittest.TestCase    ):
  snake_case_ 				=				PreTrainedTokenizerFast
  snake_case_ 				=				False
  snake_case_ 				=				True
  snake_case_ 				=				"""tokenizer_file"""

  def       __lowercase				(     self						:	Union[str, Any]			):






   '''simple docstring'''
   UpperCAmelCase__    :     Dict          =    False  # because we don't have pretrained_vocab_files_map
   super().setUp()
   UpperCAmelCase__    :     Optional[Any]          =    True

   UpperCAmelCase__    :     int          =    ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
   UpperCAmelCase__    :     Union[str, Any]          =    """SaulLu/dummy-tokenizer-bytelevel-bpe"""

   # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
   UpperCAmelCase__    :     List[str]          =    [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

   UpperCAmelCase__    :     Optional[Any]          =    PreTrainedTokenizerFast.from_pretrained(model_paths[0]			)
   tokenizer.save_pretrained(self.tmpdirname			)

  def       __lowercase				(     self						:	str			):






   '''simple docstring'''
   # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
   # model
   pass

  def       __lowercase				(     self						:	Dict			):






   '''simple docstring'''
   # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
   # model
   pass

  def       __lowercase				(     self						:	int			):






   '''simple docstring'''
   # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
   # model
   pass

  def       __lowercase				(     self						:	int			):






   '''simple docstring'''
   # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
   pass

  def       __lowercase				(     self						:	Optional[Any]			):






   '''simple docstring'''
   UpperCAmelCase__    :     Optional[Any]          =    self.tmpdirname
   # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
    with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"			):
     try:
      UpperCAmelCase__    :     Optional[int]          =    tempfile.mkdtemp()
      UpperCAmelCase__    :     Tuple          =    self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__    ,**lowerCAmelCase__			)

      tokenizer.save_pretrained(self.tmpdirname			)
      super().test_training_new_tokenizer()
     finally:
      # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
      # is restored
      shutil.rmtree(self.tmpdirname			)
      UpperCAmelCase__    :     Optional[int]          =    tmpdirname_orig

  def       __lowercase				(     self						:	Union[str, Any]			):






   '''simple docstring'''
   UpperCAmelCase__    :     int          =    self.tmpdirname
   # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
    with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"			):
     try:
      UpperCAmelCase__    :     Any          =    tempfile.mkdtemp()
      UpperCAmelCase__    :     Union[str, Any]          =    self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__    ,**lowerCAmelCase__			)

      tokenizer.save_pretrained(self.tmpdirname			)
      super().test_training_new_tokenizer_with_special_tokens_change()
     finally:
      # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
      # is restored
      shutil.rmtree(self.tmpdirname			)
      UpperCAmelCase__    :     Dict          =    tmpdirname_orig




  def       __lowercase				(     self						:	Optional[int]			):






   '''simple docstring'''
   UpperCAmelCase__    :     List[Any]          =    self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name			)

   UpperCAmelCase__    :     Dict          =    ("""a""" for _ in range(1_000			))
   UpperCAmelCase__    :     Optional[int]          =    tokenizer.train_new_from_iterator(text_iterator=lowerCAmelCase__    ,length=1_000    ,vocab_size=50			)

   UpperCAmelCase__    :     int          =    new_tokenizer.encode("""a🤗"""			)
   self.assertEqual(lowerCAmelCase__    ,[64, 172, 253, 97, 245]			)







@require_tokenizers
class    __lowercase	(			unittest.TestCase    ):

  def       __lowercase				(     self						:	Tuple			):






   '''simple docstring'''
   UpperCAmelCase__    :     Any          =    AutoTokenizer.from_pretrained("""bert-base-cased"""			)
   UpperCAmelCase__    :     Dict          =    json.loads(tokenizer._tokenizer.to_str()			)
   UpperCAmelCase__    :     List[Any]          =    len(lowerCAmelCase__			)

   with tempfile.TemporaryDirectory() as tmp_dir:
    # Hack to save this in the tokenizer_config.json
    UpperCAmelCase__    :     Union[str, Any]          =    ["""tokenizer.4.0.0.json"""]
    tokenizer.save_pretrained(lowerCAmelCase__			)
    json.dump(lowerCAmelCase__    ,open(os.path.join(lowerCAmelCase__    ,"""tokenizer.4.0.0.json"""			)    ,"""w"""			)			)

    # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
    UpperCAmelCase__    :     Optional[int]          =    AutoTokenizer.from_pretrained(lowerCAmelCase__			)
    self.assertEqual(len(lowerCAmelCase__			)    ,len(lowerCAmelCase__			) + 1			)
    UpperCAmelCase__    :     Tuple          =    json.loads(new_tokenizer._tokenizer.to_str()			)
    self.assertIn("""huggingface"""    ,json_tokenizer["""model"""]["""vocab"""]			)

    # Will need to be adjusted if we reach v42 and this test is still here.
    # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
    shutil.move(os.path.join(lowerCAmelCase__    ,"""tokenizer.4.0.0.json"""			)    ,os.path.join(lowerCAmelCase__    ,"""tokenizer.42.0.0.json"""			)			)
    UpperCAmelCase__    :     Tuple          =    ["""tokenizer.42.0.0.json"""]
    tokenizer.save_pretrained(lowerCAmelCase__			)
    UpperCAmelCase__    :     str          =    AutoTokenizer.from_pretrained(lowerCAmelCase__			)
    self.assertEqual(len(lowerCAmelCase__			)    ,len(lowerCAmelCase__			)			)
    UpperCAmelCase__    :     Union[str, Any]          =    json.loads(new_tokenizer._tokenizer.to_str()			)
    self.assertNotIn("""huggingface"""    ,json_tokenizer["""model"""]["""vocab"""]			)




  def       __lowercase				(     self						:	Union[str, Any]			):






   '''simple docstring'''
   # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
   UpperCAmelCase__    :     List[Any]          =    """hf-internal-testing/test-two-tokenizers"""

   # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
   UpperCAmelCase__    :     Dict          =    AutoTokenizer.from_pretrained(lowerCAmelCase__			)
   self.assertEqual(len(lowerCAmelCase__			)    ,28_997			)
   UpperCAmelCase__    :     List[str]          =    json.loads(tokenizer._tokenizer.to_str()			)
   self.assertIn("""huggingface"""    ,json_tokenizer["""model"""]["""vocab"""]			)

   # Testing an older version by monkey-patching the version in the module it's used.
   import transformers as old_transformers

   UpperCAmelCase__    :     Optional[Any]          =    """3.0.0"""
   UpperCAmelCase__    :     str          =    old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCAmelCase__			)
   self.assertEqual(len(lowerCAmelCase__			)    ,28_996			)
   UpperCAmelCase__    :     Optional[int]          =    json.loads(old_tokenizer._tokenizer.to_str()			)
   self.assertNotIn("""huggingface"""    ,json_tokenizer["""model"""]["""vocab"""]			)







@require_tokenizers
class    __lowercase	(			unittest.TestCase    ):

  def       __lowercase				(     self						:	Any			):






   '''simple docstring'''
   # See https://github.com/huggingface/transformers/pull/12550
   # and https://github.com/huggingface/tokenizers/issues/537
   UpperCAmelCase__    :     int          =    PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""			)
   UpperCAmelCase__    :     str          =    """The Matrix is a 1999 science fiction action film."""

   with concurrent.futures.ThreadPoolExecutor() as executor:
    UpperCAmelCase__    :     List[Any]          =    [executor.submit(self.fetch    ,lowerCAmelCase__    ,lowerCAmelCase__			) for i in range(10			)]
    UpperCAmelCase__    :     Any          =    [future.result() for future in futures]
    self.assertEqual(lowerCAmelCase__    ,[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10			)]			)




  def       __lowercase				(     self						:	Any    ,A						:	str    ,A						:	List[Any]			):






   '''simple docstring'''
   return tokenizer.encode(lowerCAmelCase__    ,truncation="""longest_first"""    ,padding="""longest"""			)







'''simple docstring'''

from __future__ import annotations







def __A  (    lowerCAmelCase_       ):
     _UpperCAmelCase					:  Optional[int]         =    {1}
     _UpperCAmelCase					:  Union[str, Any]         =    [1]
     for _ in range(2 ,       depth + 1       ):
          _UpperCAmelCase					:  Dict         =    [*previous_coefficients, 0]
          _UpperCAmelCase					:  int         =    [0, *previous_coefficients]
          _UpperCAmelCase					:  List[Any]         =    []
          for x, y in zip(lowerCAmelCase_ ,       lowerCAmelCase_       ):
               coefficients.add(x + y       )
               previous_coefficients.append(x + y       )
     return coefficients







def __A  (    lowerCAmelCase_       ):
     _UpperCAmelCase					:  Dict         =    set()
     for number in unique_coefficients:
          _UpperCAmelCase					:  Tuple         =    2
          _UpperCAmelCase					:  List[str]         =    number
          while divisor**2 <= copy_number:
               _UpperCAmelCase					:  Tuple         =    0
               while copy_number % divisor == 0:
                    copy_number //= divisor
                    multiplicity += 1
               if multiplicity >= 2:
                    non_squarefrees.add(lowerCAmelCase_       )
                    break
               divisor += 1

     return unique_coefficients.difference(lowerCAmelCase_       )







def __A  (    lowerCAmelCase_ = 51       ):
     _UpperCAmelCase					:  List[str]         =    get_pascal_triangle_unique_coefficients(lowerCAmelCase_       )
     _UpperCAmelCase					:  Optional[Any]         =    get_squarefrees(lowerCAmelCase_       )
     return sum(lowerCAmelCase_       )


if __name__ == "__main__":
   print(F"{solution() = }")


