





import inspect
import unittest

import numpy as np

from tests.test_modeling_common import floats_tensor
from transformers import MaskaFormerConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import MaskaFormerForUniversalSegmentation, MaskaFormerModel

       if is_vision_available():
              from transformers import MaskaFormerImageProcessor

if is_vision_available():
       from PIL import Image

class  __A:

     def __init__(      self,       A,       A=2,       A=True,       A=False,       A=10,       A=3,       A=32 * 8,       A=32 * 8,       A=4,       A=64,       ):





            """simple docstring"""

            _UpperCamelCase              =						parent
            _UpperCamelCase              =						batch_size
            _UpperCamelCase              =						is_training
            _UpperCamelCase              =						use_auxiliary_loss
            _UpperCamelCase              =						num_queries
            _UpperCamelCase              =						num_channels
            _UpperCamelCase              =						min_size
            _UpperCamelCase              =						max_size
            _UpperCamelCase              =						num_labels
            _UpperCamelCase              =						hidden_dim
            _UpperCamelCase              =						hidden_dim

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						floats_tensor([self.batch_size, self.num_channels, self.min_size, self.max_size]      ).to(
                A      )

            _UpperCamelCase              =						torch.ones([self.batch_size, self.min_size, self.max_size],       device=A      )

            _UpperCamelCase              =						(
                torch.rand([self.batch_size, self.num_labels, self.min_size, self.max_size],       device=A      ) > 0.5
            ).float()
            _UpperCamelCase              =						(torch.rand((self.batch_size, self.num_labels),       device=A      ) > 0.5).long()

            _UpperCamelCase              =						self.get_config()
            return config, pixel_values, pixel_mask, mask_labels, class_labels

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						MaskaFormerConfig(
                hidden_size=self.hidden_dim,       )
            _UpperCamelCase              =						self.num_queries
            _UpperCamelCase              =						self.num_labels

            _UpperCamelCase              =						[1, 1, 1, 1]
            _UpperCamelCase              =						self.num_channels

            _UpperCamelCase              =						64
            _UpperCamelCase              =						128
            _UpperCamelCase              =						self.hidden_dim
            _UpperCamelCase              =						self.hidden_dim
            _UpperCamelCase              =						self.hidden_dim
            return config

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase       ,		_UpperCamelCase       ,		_UpperCamelCase       ,		_UpperCamelCase       ,		_UpperCamelCase              =						self.prepare_config_and_inputs()
            _UpperCamelCase              =						{'''pixel_values''': pixel_values, '''pixel_mask''': pixel_mask}
            return config, inputs_dict

     def 	_UpperCamelCase     (      self,       A,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						output.encoder_hidden_states
            _UpperCamelCase              =						output.pixel_decoder_hidden_states
            _UpperCamelCase              =						output.transformer_decoder_hidden_states

            self.parent.assertTrue(len(A      ),       len(config.backbone_config.depths      )      )
            self.parent.assertTrue(len(A      ),       len(config.backbone_config.depths      )      )
            self.parent.assertTrue(len(A      ),       config.decoder_layers      )

     def 	_UpperCamelCase     (      self,       A,       A,       A,       A=False      ):





            """simple docstring"""

            with torch.no_grad():
                   _UpperCamelCase              =						MaskaFormerModel(config=A      )
                   model.to(A      )
                   model.eval()

                   _UpperCamelCase              =						model(pixel_values=A,       pixel_mask=A      )
                   _UpperCamelCase              =						model(A,       output_hidden_states=A      )

            self.parent.assertEqual(
                output.transformer_decoder_last_hidden_state.shape,       (self.batch_size, self.num_queries, self.hidden_dim),       )
            # let's ensure the other two hidden state exists
            self.parent.assertTrue(output.pixel_decoder_last_hidden_state is not None      )
            self.parent.assertTrue(output.encoder_last_hidden_state is not None      )

            if output_hidden_states:
                   self.check_output_hidden_state(A,       A      )



     def 	_UpperCamelCase     (      self,       A,       A,       A,       A,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						MaskaFormerForUniversalSegmentation(config=A      )
            model.to(A      )
            model.eval()

            def comm_check_on_output(A      ):
                   # let's still check that all the required stuff is there
                   self.parent.assertTrue(result.transformer_decoder_last_hidden_state is not None      )
                   self.parent.assertTrue(result.pixel_decoder_last_hidden_state is not None      )
                   self.parent.assertTrue(result.encoder_last_hidden_state is not None      )
                   # okay, now we need to check the logits shape
                   # due to the encoder compression, masks have a //4 spatial size
                   self.parent.assertEqual(
                       result.masks_queries_logits.shape,       (self.batch_size, self.num_queries, self.min_size // 4, self.max_size // 4),       )
                   # + 1 for null class
                   self.parent.assertEqual(
                       result.class_queries_logits.shape,       (self.batch_size, self.num_queries, self.num_labels + 1)      )

            with torch.no_grad():
                   _UpperCamelCase              =						model(pixel_values=A,       pixel_mask=A      )
                   _UpperCamelCase              =						model(A      )

                   comm_check_on_output(A      )

                   _UpperCamelCase              =						model(
                       pixel_values=A,       pixel_mask=A,       mask_labels=A,       class_labels=A      )

            comm_check_on_output(A      )

            self.parent.assertTrue(result.loss is not None      )
            self.parent.assertEqual(result.loss.shape,       torch.Size([1]      )      )






@require_torch
class  __A(						__UpperCAmelCase						,			__UpperCAmelCase						,			unittest.TestCase ):
     __A	  =       (MaskaFormerModel, MaskaFormerForUniversalSegmentation) if is_torch_available() else ()
     __A	  =       {"feature-extraction": MaskaFormerModel} if is_torch_available() else {}

     __A	  =       False
     __A	  =       False
     __A	  =       False
     __A	  =       False

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						MaskaFormerModelTester(self      )
            _UpperCamelCase              =						ConfigTester(self,       config_class=A,       has_text_modality=A      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            self.config_tester.run_common_tests()

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase       ,		_UpperCamelCase              =						self.model_tester.prepare_config_and_inputs_for_common()
            self.model_tester.create_and_check_maskaformer_model(A,       **A,       output_hidden_states=A      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_maskaformer_instance_segmentation_head_model(*A      )

     @unittest.skip(reason='''Mask2Former does not use inputs_embeds'''      )
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            pass

     @unittest.skip(reason='''Mask2Former does not have a get_input_embeddings method'''      )
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            pass

     @unittest.skip(reason='''Mask2Former is not a generative model'''      )
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            pass

     @unittest.skip(reason='''Mask2Former does not use token embeddings'''      )
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            pass

     @require_torch_multi_gpu
     @unittest.skip(
         reason='''Mask2Former has some layers using `add_module` which doesn\'t work well with `nn.DataParallel`'''      )
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            pass

     @unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''      )
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            pass

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase       ,		_UpperCamelCase              =						self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                   _UpperCamelCase              =						model_class(A      )
                   _UpperCamelCase              =						inspect.signature(model.forward      )
                   # signature.parameters is an OrderedDict => so arg_names order is deterministic
                   _UpperCamelCase              =						[*signature.parameters.keys()]

                   _UpperCamelCase              =						['''pixel_values''']
                   self.assertListEqual(arg_names[:1],       A      )

     @slow
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            for model_name in ["facebook/mask2former-swin-small-coco-instance"]:
                   _UpperCamelCase              =						MaskaFormerModel.from_pretrained(A      )
                   self.assertIsNotNone(A      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						(self.model_tester.min_size,) * 2
            _UpperCamelCase              =						{
                '''pixel_values''': torch.randn((2, 3, *size),       device=A      ),
                '''mask_labels''': torch.randn((2, 10, *size),       device=A      ),
                '''class_labels''': torch.zeros(2,       10,       device=A      ).long(),
            }
            _UpperCamelCase              =						self.model_tester.get_config()

            _UpperCamelCase              =						MaskaFormerForUniversalSegmentation(A      ).to(A      )
            _UpperCamelCase              =						model(**A      )
            self.assertTrue(outputs.loss is not None      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase       ,		_UpperCamelCase              =						self.model_tester.prepare_config_and_inputs_for_common()
            self.model_tester.create_and_check_maskaformer_model(A,       **A,       output_hidden_states=A      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase       ,		_UpperCamelCase              =						self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                   _UpperCamelCase              =						model_class(A      ).to(A      )
                   _UpperCamelCase              =						model(**A,       output_attentions=A      )
                   self.assertTrue(outputs.attentions is not None      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            if not self.model_tester.is_training:
                   return

            _UpperCamelCase              =						self.all_model_classes[1]
            _UpperCamelCase       ,		_UpperCamelCase       ,		_UpperCamelCase       ,		_UpperCamelCase       ,		_UpperCamelCase              =						self.model_tester.prepare_config_and_inputs()

            _UpperCamelCase              =						model_class(A      )
            model.to(A      )
            model.train()

            _UpperCamelCase              =						model(A,       mask_labels=A,       class_labels=A      ).loss
            loss.backward()



     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self.all_model_classes[1]
            _UpperCamelCase       ,		_UpperCamelCase       ,		_UpperCamelCase       ,		_UpperCamelCase       ,		_UpperCamelCase              =						self.model_tester.prepare_config_and_inputs()
            _UpperCamelCase              =						True
            _UpperCamelCase              =						True

            _UpperCamelCase              =						model_class(A      ).to(A      )
            model.train()

            _UpperCamelCase              =						model(A,       mask_labels=A,       class_labels=A      )

            _UpperCamelCase              =						outputs.encoder_hidden_states[0]
            encoder_hidden_states.retain_grad()

            _UpperCamelCase              =						outputs.pixel_decoder_hidden_states[0]
            pixel_decoder_hidden_states.retain_grad()

            _UpperCamelCase              =						outputs.transformer_decoder_hidden_states[0]
            transformer_decoder_hidden_states.retain_grad()

            _UpperCamelCase              =						outputs.attentions[0]
            attentions.retain_grad()

            outputs.loss.backward(retain_graph=A      )

            self.assertIsNotNone(encoder_hidden_states.grad      )
            self.assertIsNotNone(pixel_decoder_hidden_states.grad      )
            self.assertIsNotNone(transformer_decoder_hidden_states.grad      )
            self.assertIsNotNone(attentions.grad      )






lowercase    : Tuple    	=  1e-4


def     SCREAMING_SNAKE_CASE			(      ):
       _UpperCamelCase              =						Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''					)
       return image

@require_vision
@slow
class  __A(						unittest.TestCase ):

     @cached_property
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            return "facebook/mask2former-swin-small-coco-instance"

     @cached_property
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            return MaskaFormerImageProcessor.from_pretrained(self.model_checkpoints      ) if is_vision_available() else None

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						MaskaFormerModel.from_pretrained(self.model_checkpoints      ).to(A      )
            _UpperCamelCase              =						self.default_image_processor
            _UpperCamelCase              =						prepare_img()
            _UpperCamelCase              =						image_processor(A,       return_tensors='''pt'''      ).to(A      )
            _UpperCamelCase              =						inputs['''pixel_values'''].shape
            # check size is divisible by 32
            self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0      )
            # check size
            self.assertEqual(A,       (1, 3, 384, 384)      )

            with torch.no_grad():
                   _UpperCamelCase              =						model(**A      )

            _UpperCamelCase              =						torch.tensor(
                [[-0.2_790, -1.0_717, -1.1_668], [-0.5_128, -0.3_128, -0.4_987], [-0.5_832, 0.1_971, -0.0_197]]      ).to(A      )
            self.assertTrue(
                torch.allclose(
                    outputs.encoder_last_hidden_state[0, 0, :3, :3],       A,       atol=A      )      )

            _UpperCamelCase              =						torch.tensor(
                [[0.8_973, 1.1_847, 1.1_776], [1.1_934, 1.5_040, 1.5_128], [1.1_153, 1.4_486, 1.4_951]]      ).to(A      )
            self.assertTrue(
                torch.allclose(
                    outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3],       A,       atol=A      )      )

            _UpperCamelCase              =						torch.tensor(
                [[2.1_152, 1.7_000, -0.8_603], [1.5_808, 1.8_004, -0.9_353], [1.6_043, 1.7_495, -0.5_999]]      ).to(A      )
            self.assertTrue(
                torch.allclose(
                    outputs.transformer_decoder_last_hidden_state[0, :3, :3],       A,       atol=A      )      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						MaskaFormerForUniversalSegmentation.from_pretrained(self.model_checkpoints      ).to(A      ).eval()
            _UpperCamelCase              =						self.default_image_processor
            _UpperCamelCase              =						prepare_img()
            _UpperCamelCase              =						image_processor(A,       return_tensors='''pt'''      ).to(A      )
            _UpperCamelCase              =						inputs['''pixel_values'''].shape
            # check size is divisible by 32
            self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0      )
            # check size
            self.assertEqual(A,       (1, 3, 384, 384)      )

            with torch.no_grad():
                   _UpperCamelCase              =						model(**A      )
            # masks_queries_logits
            _UpperCamelCase              =						outputs.masks_queries_logits
            self.assertEqual(
                masks_queries_logits.shape,       (1, model.config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)      )
            _UpperCamelCase              =						[
                [-8.7_839, -9.0_056, -8.8_121],
                [-7.4_104, -7.0_313, -6.5_401],
                [-6.6_105, -6.3_427, -6.4_675],
            ]
            _UpperCamelCase              =						torch.tensor(A      ).to(A      )
            self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3],       A,       atol=A      )      )
            # class_queries_logits
            _UpperCamelCase              =						outputs.class_queries_logits
            self.assertEqual(class_queries_logits.shape,       (1, model.config.num_queries, model.config.num_labels + 1)      )
            _UpperCamelCase              =						torch.tensor(
                [
                    [1.8_324, -8.0_835, -4.1_922],
                    [0.8_450, -9.0_050, -3.6_053],
                    [0.3_045, -7.7_293, -3.0_275],
                ]      ).to(A      )
            self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3],       A,       atol=A      )      )



     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						MaskaFormerForUniversalSegmentation.from_pretrained(self.model_checkpoints      ).to(A      ).eval()
            _UpperCamelCase              =						self.default_image_processor

            _UpperCamelCase              =						image_processor(
                [np.zeros((3, 800, 1333)      ), np.zeros((3, 800, 1333)      )],       segmentation_maps=[np.zeros((384, 384)      ).astype(np.floataa      ), np.zeros((384, 384)      ).astype(np.floataa      )],       return_tensors='''pt''',       )

            _UpperCamelCase              =						inputs['''pixel_values'''].to(A      )
            _UpperCamelCase              =						[el.to(A      ) for el in inputs['''mask_labels''']]
            _UpperCamelCase              =						[el.to(A      ) for el in inputs['''class_labels''']]

            with torch.no_grad():
                   _UpperCamelCase              =						model(**A      )

            self.assertTrue(outputs.loss is not None      )













import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
       from test_module.custom_tokenization_fast import CustomTokenizerFast

class  __A(						unittest.TestCase ):

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						mock.Mock()
            _UpperCamelCase              =						500
            _UpperCamelCase              =						{}
            _UpperCamelCase              =						HTTPError
            _UpperCamelCase              =						{}

            # Download this model to make sure it's in the cache.
            _UpperCamelCase              =						BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''      )

            # Under the mock environment we get a 500 error when trying to reach the tokenizer.
            with mock.patch('''requests.Session.request''',       return_value=A      ) as mock_head:
                   _UpperCamelCase              =						BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''      )
                   # This check we did call the fake head request
                   mock_head.assert_called()

     @require_tokenizers
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						mock.Mock()
            _UpperCamelCase              =						500
            _UpperCamelCase              =						{}
            _UpperCamelCase              =						HTTPError
            _UpperCamelCase              =						{}

            # Download this model to make sure it's in the cache.
            _UpperCamelCase              =						GPTaTokenizerFast.from_pretrained('''gpt2'''      )

            # Under the mock environment we get a 500 error when trying to reach the tokenizer.
            with mock.patch('''requests.Session.request''',       return_value=A      ) as mock_head:
                   _UpperCamelCase              =						GPTaTokenizerFast.from_pretrained('''gpt2'''      )
                   # This check we did call the fake head request
                   mock_head.assert_called()

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            try:
                   _UpperCamelCase              =						tempfile.mktemp()
                   with open(A,       '''wb'''      ) as f:
                          http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',       A      )

                   _UpperCamelCase              =						AlbertTokenizer.from_pretrained(A      )
            finally:
                   os.remove(A      )

            # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
            # the current folder and have the right name.
            if os.path.isfile('''tokenizer.json'''      ):
                   # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
                   return
            try:
                   with open('''tokenizer.json''',       '''wb'''      ) as f:
                          http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json''',       A      )
                   _UpperCamelCase              =						AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''      )
                   # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
                   self.assertEqual(tokenizer.vocab_size,       1000      )
                   # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

            finally:
                   os.remove('''tokenizer.json'''      )



     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''      )






@is_staging_test
class  __A(						unittest.TestCase ):
     __A	  =       ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]

     @classmethod
     def 	_UpperCamelCase     (      cls      ):





            """simple docstring"""

            _UpperCamelCase              =						TOKEN
            HfFolder.save_token(A      )

     @classmethod
     def 	_UpperCamelCase     (      cls      ):





            """simple docstring"""

            try:
                   delete_repo(token=cls._token,       repo_id='''test-tokenizer'''      )
            except HTTPError:
                   pass

            try:
                   delete_repo(token=cls._token,       repo_id='''valid_org/test-tokenizer-org'''      )
            except HTTPError:
                   pass

            try:
                   delete_repo(token=cls._token,       repo_id='''test-dynamic-tokenizer'''      )
            except HTTPError:
                   pass

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            with tempfile.TemporaryDirectory() as tmp_dir:
                   _UpperCamelCase              =						os.path.join(A,       '''vocab.txt'''      )
                   with open(A,       '''w''',       encoding='''utf-8'''      ) as vocab_writer:
                          vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )
                   _UpperCamelCase              =						BertTokenizer(A      )

            tokenizer.push_to_hub('''test-tokenizer''',       use_auth_token=self._token      )
            _UpperCamelCase              =						BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''      )
            self.assertDictEqual(new_tokenizer.vocab,       tokenizer.vocab      )

            # Reset repo
            delete_repo(token=self._token,       repo_id='''test-tokenizer'''      )

            # Push to hub via save_pretrained
            with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer.save_pretrained(A,       repo_id='''test-tokenizer''',       push_to_hub=A,       use_auth_token=self._token      )

            _UpperCamelCase              =						BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''      )
            self.assertDictEqual(new_tokenizer.vocab,       tokenizer.vocab      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            with tempfile.TemporaryDirectory() as tmp_dir:
                   _UpperCamelCase              =						os.path.join(A,       '''vocab.txt'''      )
                   with open(A,       '''w''',       encoding='''utf-8'''      ) as vocab_writer:
                          vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )
                   _UpperCamelCase              =						BertTokenizer(A      )

            tokenizer.push_to_hub('''valid_org/test-tokenizer-org''',       use_auth_token=self._token      )
            _UpperCamelCase              =						BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''      )
            self.assertDictEqual(new_tokenizer.vocab,       tokenizer.vocab      )

            # Reset repo
            delete_repo(token=self._token,       repo_id='''valid_org/test-tokenizer-org'''      )

            # Push to hub via save_pretrained
            with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer.save_pretrained(
                       A,       repo_id='''valid_org/test-tokenizer-org''',       push_to_hub=A,       use_auth_token=self._token      )

            _UpperCamelCase              =						BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''      )
            self.assertDictEqual(new_tokenizer.vocab,       tokenizer.vocab      )



     @require_tokenizers
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            CustomTokenizer.register_for_auto_class()
            with tempfile.TemporaryDirectory() as tmp_dir:
                   _UpperCamelCase              =						os.path.join(A,       '''vocab.txt'''      )
                   with open(A,       '''w''',       encoding='''utf-8'''      ) as vocab_writer:
                          vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )
                   _UpperCamelCase              =						CustomTokenizer(A      )

            # No fast custom tokenizer
            tokenizer.push_to_hub('''test-dynamic-tokenizer''',       use_auth_token=self._token      )

            _UpperCamelCase              =						AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer''',       trust_remote_code=A      )
            # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
            self.assertEqual(tokenizer.__class__.__name__,       '''CustomTokenizer'''      )

            # Fast and slow custom tokenizer
            CustomTokenizerFast.register_for_auto_class()
            with tempfile.TemporaryDirectory() as tmp_dir:
                   _UpperCamelCase              =						os.path.join(A,       '''vocab.txt'''      )
                   with open(A,       '''w''',       encoding='''utf-8'''      ) as vocab_writer:
                          vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )

                   _UpperCamelCase              =						BertTokenizerFast.from_pretrained(A      )
                   bert_tokenizer.save_pretrained(A      )
                   _UpperCamelCase              =						CustomTokenizerFast.from_pretrained(A      )

            tokenizer.push_to_hub('''test-dynamic-tokenizer''',       use_auth_token=self._token      )

            _UpperCamelCase              =						AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer''',       trust_remote_code=A      )
            # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
            self.assertEqual(tokenizer.__class__.__name__,       '''CustomTokenizerFast'''      )
            _UpperCamelCase              =						AutoTokenizer.from_pretrained(
                F'''{USER}/test-dynamic-tokenizer''',       use_fast=A,       trust_remote_code=A      )
            # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
            self.assertEqual(tokenizer.__class__.__name__,       '''CustomTokenizer'''      )






class  __A(						unittest.TestCase ):

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						Trie()
            trie.add('''Hello 友達'''      )
            self.assertEqual(trie.data,       {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}      )
            trie.add('''Hello'''      )
            trie.data
            self.assertEqual(trie.data,       {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						Trie()
            self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''      ),       ['''[CLS] This is a extra_id_100''']      )
            trie.add('''[CLS]'''      )
            trie.add('''extra_id_1'''      )
            trie.add('''extra_id_100'''      )
            self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''      ),       ['''[CLS]''', ''' This is a ''', '''extra_id_100''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						Trie()
            trie.add('''A'''      )
            self.assertEqual(trie.split('''ABC'''      ),       ['''A''', '''BC''']      )
            self.assertEqual(trie.split('''BCA'''      ),       ['''BC''', '''A''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						Trie()
            trie.add('''TOKEN]'''      )
            trie.add('''[SPECIAL_TOKEN]'''      )
            self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''      ),       ['''This is something ''', '''[SPECIAL_TOKEN]''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						Trie()
            trie.add('''A'''      )
            trie.add('''P'''      )
            trie.add('''[SPECIAL_TOKEN]'''      )
            self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''      ),       ['''This is something ''', '''[SPECIAL_TOKEN]''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						Trie()
            trie.add('''AB'''      )
            trie.add('''B'''      )
            trie.add('''C'''      )
            self.assertEqual(trie.split('''ABC'''      ),       ['''AB''', '''C''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						Trie()
            trie.add('''ABC'''      )
            trie.add('''B'''      )
            trie.add('''CD'''      )
            self.assertEqual(trie.split('''ABCD'''      ),       ['''ABC''', '''D''']      )



     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						Trie()
            _UpperCamelCase              =						trie.cut_text('''ABC''',       [0, 0, 2, 1, 2, 3]      )
            self.assertEqual(A,       ['''AB''', '''C''']      )






