


import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    Pipeline,
    ZeroShotClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
__UpperCamelCase       :       Dict						       =				{"LayoutLMv2Config", "LayoutLMv3Config"}

@is_pipeline_test
class       __magic_name__	(     unittest.TestCase):
     A:		Union[str, Any]			=      MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
     A:		str			=      TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

     if model_mapping is not None:
          A:		Any			=      {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
     if tf_model_mapping is not None:
          A:		Optional[Any]			=      {
              config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
          }


     def      UpperCAmelCase__	(      self							:     Optional[Any]							,	lowerCamelCase__							:     int							,	lowerCamelCase__							:     Union[str, Any]							,	lowerCamelCase__							:     List[Any]			)  ->      List[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   ZeroShotClassificationPipeline(
          model=lowerCamelCase__							,	tokenizer=lowerCamelCase__							,	candidate_labels=['''polics''', '''health''']			)
      return classifier, ["Who are you voting for in 2020?", "My stomach hurts."]


     def      UpperCAmelCase__	(      self							:     str							,	lowerCamelCase__							:     Union[str, Any]							,	lowerCamelCase__							:     Optional[int]			)  ->      List[str]:





      '''simple docstring'''






      UpperCamelCase__					:    str          =   classifier('''Who are you voting for in 2020?'''							,	candidate_labels='''politics'''			)
      self.assertEqual(lowerCamelCase__							,	{'''sequence''': ANY(lowerCamelCase__			), '''labels''': [ANY(lowerCamelCase__			)], '''scores''': [ANY(lowerCamelCase__			)]}			)

      # No kwarg
      UpperCamelCase__					:    List[str]          =   classifier('''Who are you voting for in 2020?'''							,	['''politics''']			)
      self.assertEqual(lowerCamelCase__							,	{'''sequence''': ANY(lowerCamelCase__			), '''labels''': [ANY(lowerCamelCase__			)], '''scores''': [ANY(lowerCamelCase__			)]}			)

      UpperCamelCase__					:    Optional[int]          =   classifier('''Who are you voting for in 2020?'''							,	candidate_labels=['''politics''']			)
      self.assertEqual(lowerCamelCase__							,	{'''sequence''': ANY(lowerCamelCase__			), '''labels''': [ANY(lowerCamelCase__			)], '''scores''': [ANY(lowerCamelCase__			)]}			)

      UpperCamelCase__					:    Union[str, Any]          =   classifier('''Who are you voting for in 2020?'''							,	candidate_labels='''politics, public health'''			)
      self.assertEqual(
          lowerCamelCase__							,	{'''sequence''': ANY(lowerCamelCase__			), '''labels''': [ANY(lowerCamelCase__			), ANY(lowerCamelCase__			)], '''scores''': [ANY(lowerCamelCase__			), ANY(lowerCamelCase__			)]}			)
      self.assertAlmostEqual(sum(nested_simplify(outputs['''scores''']			)			)							,	1.0			)

      UpperCamelCase__					:    List[Any]          =   classifier('''Who are you voting for in 2020?'''							,	candidate_labels=['''politics''', '''public health''']			)
      self.assertEqual(
          lowerCamelCase__							,	{'''sequence''': ANY(lowerCamelCase__			), '''labels''': [ANY(lowerCamelCase__			), ANY(lowerCamelCase__			)], '''scores''': [ANY(lowerCamelCase__			), ANY(lowerCamelCase__			)]}			)
      self.assertAlmostEqual(sum(nested_simplify(outputs['''scores''']			)			)							,	1.0			)

      UpperCamelCase__					:    Optional[int]          =   classifier(
          '''Who are you voting for in 2020?'''							,	candidate_labels='''politics'''							,	hypothesis_template='''This text is about {}'''			)
      self.assertEqual(lowerCamelCase__							,	{'''sequence''': ANY(lowerCamelCase__			), '''labels''': [ANY(lowerCamelCase__			)], '''scores''': [ANY(lowerCamelCase__			)]}			)

      # https://github.com/huggingface/transformers/issues/13846
      UpperCamelCase__					:    List[Any]          =   classifier(['''I am happy''']							,	['''positive''', '''negative''']			)
      self.assertEqual(
          lowerCamelCase__							,	[
              {'''sequence''': ANY(lowerCamelCase__			), '''labels''': [ANY(lowerCamelCase__			), ANY(lowerCamelCase__			)], '''scores''': [ANY(lowerCamelCase__			), ANY(lowerCamelCase__			)]}
              for i in range(1			)
          ]							,	)
      UpperCamelCase__					:    Optional[Any]          =   classifier(['''I am happy''', '''I am sad''']							,	['''positive''', '''negative''']			)
      self.assertEqual(
          lowerCamelCase__							,	[
              {'''sequence''': ANY(lowerCamelCase__			), '''labels''': [ANY(lowerCamelCase__			), ANY(lowerCamelCase__			)], '''scores''': [ANY(lowerCamelCase__			), ANY(lowerCamelCase__			)]}
              for i in range(2			)
          ]							,	)

      with self.assertRaises(lowerCamelCase__			):
       classifier(''''''							,	candidate_labels='''politics'''			)

      with self.assertRaises(lowerCamelCase__			):
       classifier(lowerCamelCase__							,	candidate_labels='''politics'''			)

      with self.assertRaises(lowerCamelCase__			):
       classifier('''Who are you voting for in 2020?'''							,	candidate_labels=''''''			)

      with self.assertRaises(lowerCamelCase__			):
       classifier('''Who are you voting for in 2020?'''							,	candidate_labels=lowerCamelCase__			)

      with self.assertRaises(lowerCamelCase__			):
       classifier(
           '''Who are you voting for in 2020?'''							,	candidate_labels='''politics'''							,	hypothesis_template='''Not formatting template'''							,	)

      with self.assertRaises(lowerCamelCase__			):
       classifier(
           '''Who are you voting for in 2020?'''							,	candidate_labels='''politics'''							,	hypothesis_template=lowerCamelCase__							,	)

      self.run_entailment_id(lowerCamelCase__			)


     def      UpperCAmelCase__	(      self							:     Optional[Any]							,	lowerCamelCase__							:     Pipeline			)  ->      List[str]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   zero_shot_classifier.model.config
      UpperCamelCase__					:    List[Any]          =   config.labelaid
      UpperCamelCase__					:    Tuple          =   zero_shot_classifier.entailment_id

      UpperCamelCase__					:    Optional[int]          =   {'''LABEL_0''': 0, '''LABEL_1''': 1, '''LABEL_2''': 2}
      self.assertEqual(zero_shot_classifier.entailment_id							,	-1			)

      UpperCamelCase__					:    Optional[Any]          =   {'''entailment''': 0, '''neutral''': 1, '''contradiction''': 2}
      self.assertEqual(zero_shot_classifier.entailment_id							,	0			)

      UpperCamelCase__					:    Optional[Any]          =   {'''ENTAIL''': 0, '''NON-ENTAIL''': 1}
      self.assertEqual(zero_shot_classifier.entailment_id							,	0			)

      UpperCamelCase__					:    Dict          =   {'''ENTAIL''': 2, '''NEUTRAL''': 1, '''CONTR''': 0}
      self.assertEqual(zero_shot_classifier.entailment_id							,	2			)

      UpperCamelCase__					:    Any          =   original_labelaid
      self.assertEqual(lowerCamelCase__							,	zero_shot_classifier.entailment_id			)


     @require_torch
     def      UpperCAmelCase__	(      self							:     Any			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    Optional[Any]          =   pipeline(
          '''zero-shot-classification'''							,	model='''sshleifer/tiny-distilbert-base-cased-distilled-squad'''							,	framework='''pt'''							,	)
      # There was a regression in 4.10 for this
      # Adding a test so we don't make the mistake again.
      # https://github.com/huggingface/transformers/issues/13381#issuecomment-912343499
      zero_shot_classifier(
          '''Who are you voting for in 2020?''' * 100							,	candidate_labels=['''politics''', '''public health''', '''science''']			)


     @require_torch
     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      Tuple:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   pipeline(
          '''zero-shot-classification'''							,	model='''sshleifer/tiny-distilbert-base-cased-distilled-squad'''							,	framework='''pt'''							,	)
      UpperCamelCase__					:    int          =   zero_shot_classifier(
          '''Who are you voting for in 2020?'''							,	candidate_labels=['''politics''', '''public health''', '''science''']			)

      self.assertEqual(
          nested_simplify(lowerCamelCase__			)							,	{
              '''sequence''': '''Who are you voting for in 2020?''',
              '''labels''': ['''science''', '''public health''', '''politics'''],
              '''scores''': [0.333, 0.333, 0.333],
          }							,	)


     @require_tf
     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      List[str]:





      '''simple docstring'''






      UpperCamelCase__					:    str          =   pipeline(
          '''zero-shot-classification'''							,	model='''sshleifer/tiny-distilbert-base-cased-distilled-squad'''							,	framework='''tf'''							,	)
      UpperCamelCase__					:    Union[str, Any]          =   zero_shot_classifier(
          '''Who are you voting for in 2020?'''							,	candidate_labels=['''politics''', '''public health''', '''science''']			)

      self.assertEqual(
          nested_simplify(lowerCamelCase__			)							,	{
              '''sequence''': '''Who are you voting for in 2020?''',
              '''labels''': ['''science''', '''public health''', '''politics'''],
              '''scores''': [0.333, 0.333, 0.333],
          }							,	)


     @slow
     @require_torch
     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   pipeline('''zero-shot-classification'''							,	model='''roberta-large-mnli'''							,	framework='''pt'''			)
      UpperCamelCase__					:    Union[str, Any]          =   zero_shot_classifier(
          '''Who are you voting for in 2020?'''							,	candidate_labels=['''politics''', '''public health''', '''science''']			)

      self.assertEqual(
          nested_simplify(lowerCamelCase__			)							,	{
              '''sequence''': '''Who are you voting for in 2020?''',
              '''labels''': ['''politics''', '''public health''', '''science'''],
              '''scores''': [0.976, 0.015, 0.009],
          }							,	)
      UpperCamelCase__					:    str          =   zero_shot_classifier(
          '''The dominant sequence transduction models are based on complex recurrent or convolutional neural networks'''
          ''' in an encoder-decoder configuration. The best performing models also connect the encoder and decoder'''
          ''' through an attention mechanism. We propose a new simple network architecture, the Transformer, based'''
          ''' solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two'''
          ''' machine translation tasks show these models to be superior in quality while being more parallelizable'''
          ''' and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014'''
          ''' English-to-German translation task, improving over the existing best results, including ensembles by'''
          ''' over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new'''
          ''' single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small'''
          ''' fraction of the training costs of the best models from the literature. We show that the Transformer'''
          ''' generalizes well to other tasks by applying it successfully to English constituency parsing both with'''
          ''' large and limited training data.'''							,	candidate_labels=['''machine learning''', '''statistics''', '''translation''', '''vision''']							,	multi_label=lowerCamelCase__							,	)
      self.assertEqual(
          nested_simplify(lowerCamelCase__			)							,	{
              '''sequence''': (
                  '''The dominant sequence transduction models are based on complex recurrent or convolutional neural'''
                  ''' networks in an encoder-decoder configuration. The best performing models also connect the'''
                  ''' encoder and decoder through an attention mechanism. We propose a new simple network'''
                  ''' architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence'''
                  ''' and convolutions entirely. Experiments on two machine translation tasks show these models to be'''
                  ''' superior in quality while being more parallelizable and requiring significantly less time to'''
                  ''' train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,'''
                  ''' improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014'''
                  ''' English-to-French translation task, our model establishes a new single-model state-of-the-art'''
                  ''' BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training'''
                  ''' costs of the best models from the literature. We show that the Transformer generalizes well to'''
                  ''' other tasks by applying it successfully to English constituency parsing both with large and'''
                  ''' limited training data.'''
              ),
              '''labels''': ['''translation''', '''machine learning''', '''vision''', '''statistics'''],
              '''scores''': [0.817, 0.713, 0.018, 0.018],
          }							,	)


     @slow
     @require_tf
     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    List[str]          =   pipeline('''zero-shot-classification'''							,	model='''roberta-large-mnli'''							,	framework='''tf'''			)
      UpperCamelCase__					:    Union[str, Any]          =   zero_shot_classifier(
          '''Who are you voting for in 2020?'''							,	candidate_labels=['''politics''', '''public health''', '''science''']			)

      self.assertEqual(
          nested_simplify(lowerCamelCase__			)							,	{
              '''sequence''': '''Who are you voting for in 2020?''',
              '''labels''': ['''politics''', '''public health''', '''science'''],
              '''scores''': [0.976, 0.015, 0.009],
          }							,	)
      UpperCamelCase__					:    List[str]          =   zero_shot_classifier(
          '''The dominant sequence transduction models are based on complex recurrent or convolutional neural networks'''
          ''' in an encoder-decoder configuration. The best performing models also connect the encoder and decoder'''
          ''' through an attention mechanism. We propose a new simple network architecture, the Transformer, based'''
          ''' solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two'''
          ''' machine translation tasks show these models to be superior in quality while being more parallelizable'''
          ''' and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014'''
          ''' English-to-German translation task, improving over the existing best results, including ensembles by'''
          ''' over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new'''
          ''' single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small'''
          ''' fraction of the training costs of the best models from the literature. We show that the Transformer'''
          ''' generalizes well to other tasks by applying it successfully to English constituency parsing both with'''
          ''' large and limited training data.'''							,	candidate_labels=['''machine learning''', '''statistics''', '''translation''', '''vision''']							,	multi_label=lowerCamelCase__							,	)
      self.assertEqual(
          nested_simplify(lowerCamelCase__			)							,	{
              '''sequence''': (
                  '''The dominant sequence transduction models are based on complex recurrent or convolutional neural'''
                  ''' networks in an encoder-decoder configuration. The best performing models also connect the'''
                  ''' encoder and decoder through an attention mechanism. We propose a new simple network'''
                  ''' architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence'''
                  ''' and convolutions entirely. Experiments on two machine translation tasks show these models to be'''
                  ''' superior in quality while being more parallelizable and requiring significantly less time to'''
                  ''' train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,'''
                  ''' improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014'''
                  ''' English-to-French translation task, our model establishes a new single-model state-of-the-art'''
                  ''' BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training'''
                  ''' costs of the best models from the literature. We show that the Transformer generalizes well to'''
                  ''' other tasks by applying it successfully to English constituency parsing both with large and'''
                  ''' limited training data.'''
              ),
              '''labels''': ['''translation''', '''machine learning''', '''vision''', '''statistics'''],
              '''scores''': [0.817, 0.713, 0.018, 0.018],
          }							,	)






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
       from test_module.custom_tokenization_fast import CustomTokenizerFast

class       __magic_name__	(     unittest.TestCase):


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   mock.Mock()
      UpperCamelCase__					:    Tuple          =   500
      UpperCamelCase__					:    Dict          =   {}
      UpperCamelCase__					:    Dict          =   HTTPError
      UpperCamelCase__					:    Optional[Any]          =   {}

      # Download this model to make sure it's in the cache.
      UpperCamelCase__					:    List[Any]          =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request'''							,	return_value=lowerCamelCase__			) as mock_head:
       UpperCamelCase__					:    Any          =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)
       # This check we did call the fake head request
       mock_head.assert_called()


     @require_tokenizers
     def      UpperCAmelCase__	(      self							:     str			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    Union[str, Any]          =   mock.Mock()
      UpperCamelCase__					:    Any          =   500
      UpperCamelCase__					:    str          =   {}
      UpperCamelCase__					:    Optional[Any]          =   HTTPError
      UpperCamelCase__					:    Tuple          =   {}

      # Download this model to make sure it's in the cache.
      UpperCamelCase__					:    Optional[Any]          =   GPTaTokenizerFast.from_pretrained('''gpt2'''			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request'''							,	return_value=lowerCamelCase__			) as mock_head:
       UpperCamelCase__					:    str          =   GPTaTokenizerFast.from_pretrained('''gpt2'''			)
       # This check we did call the fake head request
       mock_head.assert_called()


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[str]:





      '''simple docstring'''






      try:
       UpperCamelCase__					:    Dict          =   tempfile.mktemp()
       with open(lowerCamelCase__							,	'''wb'''			) as f:
        http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''							,	lowerCamelCase__			)

       UpperCamelCase__					:    Dict          =   AlbertTokenizer.from_pretrained(lowerCamelCase__			)
      finally:
       os.remove(lowerCamelCase__			)

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile('''tokenizer.json'''			):
       # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
       return
      try:
       with open('''tokenizer.json'''							,	'''wb'''			) as f:
        http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''							,	lowerCamelCase__			)
       UpperCamelCase__					:    Union[str, Any]          =   AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''			)
       # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
       self.assertEqual(tokenizer.vocab_size							,	1000			)
       # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
       os.remove('''tokenizer.json'''			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''			)



@is_staging_test
class       __magic_name__	(     unittest.TestCase):
     A:		List[Any]			=      ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]


     @classmethod
     def      UpperCAmelCase__	(      cls							:     List[str]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    List[str]          =   TOKEN
      HfFolder.save_token(lowerCamelCase__			)


     @classmethod
     def      UpperCAmelCase__	(      cls							:     Any			)  ->      Optional[Any]:





      '''simple docstring'''






      try:
       delete_repo(token=cls._token							,	repo_id='''test-tokenizer'''			)
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token							,	repo_id='''valid_org/test-tokenizer-org'''			)
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token							,	repo_id='''test-dynamic-tokenizer'''			)
      except HTTPError:
       pass


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      Optional[int]:





      '''simple docstring'''






      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    int          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    Tuple          =   BertTokenizer(lowerCamelCase__			)

      tokenizer.push_to_hub('''test-tokenizer'''							,	use_auth_token=self._token			)
      UpperCamelCase__					:    Union[str, Any]          =   BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token							,	repo_id='''test-tokenizer'''			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(lowerCamelCase__							,	repo_id='''test-tokenizer'''							,	push_to_hub=lowerCamelCase__							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      Optional[int]:





      '''simple docstring'''






      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    Dict          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    List[str]          =   BertTokenizer(lowerCamelCase__			)

      tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''							,	use_auth_token=self._token			)
      UpperCamelCase__					:    Optional[Any]          =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token							,	repo_id='''valid_org/test-tokenizer-org'''			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(
           lowerCamelCase__							,	repo_id='''valid_org/test-tokenizer-org'''							,	push_to_hub=lowerCamelCase__							,	use_auth_token=self._token			)

      UpperCamelCase__					:    Union[str, Any]          =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)


     @require_tokenizers
     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      int:





      '''simple docstring'''






      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    List[Any]          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    Any          =   CustomTokenizer(lowerCamelCase__			)

      # No fast custom tokenizer
      tokenizer.push_to_hub('''test-dynamic-tokenizer'''							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizer'''			)

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    List[Any]          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)

       UpperCamelCase__					:    List[Any]          =   BertTokenizerFast.from_pretrained(lowerCamelCase__			)
       bert_tokenizer.save_pretrained(lowerCamelCase__			)
       UpperCamelCase__					:    Optional[int]          =   CustomTokenizerFast.from_pretrained(lowerCamelCase__			)

      tokenizer.push_to_hub('''test-dynamic-tokenizer'''							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizerFast'''			)
      UpperCamelCase__					:    int          =   AutoTokenizer.from_pretrained(
          F"{USER}/test-dynamic-tokenizer"							,	use_fast=lowerCamelCase__							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizer'''			)



class       __magic_name__	(     unittest.TestCase):


     def      UpperCAmelCase__	(      self							:     str			)  ->      Dict:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   Trie()
      trie.add('''Hello 友達'''			)
      self.assertEqual(trie.data							,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}			)
      trie.add('''Hello'''			)
      trie.data
      self.assertEqual(trie.data							,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      str:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   Trie()
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''			)							,	['''[CLS] This is a extra_id_100''']			)
      trie.add('''[CLS]'''			)
      trie.add('''extra_id_1'''			)
      trie.add('''extra_id_100'''			)
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''			)							,	['''[CLS]''', ''' This is a ''', '''extra_id_100''']			)


     def      UpperCAmelCase__	(      self							:     int			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   Trie()
      trie.add('''A'''			)
      self.assertEqual(trie.split('''ABC'''			)							,	['''A''', '''BC''']			)
      self.assertEqual(trie.split('''BCA'''			)							,	['''BC''', '''A''']			)


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   Trie()
      trie.add('''TOKEN]'''			)
      trie.add('''[SPECIAL_TOKEN]'''			)
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''			)							,	['''This is something ''', '''[SPECIAL_TOKEN]''']			)


     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   Trie()
      trie.add('''A'''			)
      trie.add('''P'''			)
      trie.add('''[SPECIAL_TOKEN]'''			)
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''			)							,	['''This is something ''', '''[SPECIAL_TOKEN]''']			)


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[str]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   Trie()
      trie.add('''AB'''			)
      trie.add('''B'''			)
      trie.add('''C'''			)
      self.assertEqual(trie.split('''ABC'''			)							,	['''AB''', '''C''']			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   Trie()
      trie.add('''ABC'''			)
      trie.add('''B'''			)
      trie.add('''CD'''			)
      self.assertEqual(trie.split('''ABCD'''			)							,	['''ABC''', '''D''']			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      Union[str, Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   Trie()
      UpperCamelCase__					:    Optional[Any]          =   trie.cut_text('''ABC'''							,	[0, 0, 2, 1, 2, 3]			)
      self.assertEqual(lowerCamelCase__							,	['''AB''', '''C''']			)


