


from __future__ import annotations

import inspect
import unittest

from transformers import MobileViTConfig
from transformers.file_utils import is_tf_available, is_vision_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import numpy as np
							import tensorflow as tf

							from transformers import TFMobileViTForImageClassification, TFMobileViTForSemanticSegmentation, TFMobileViTModel
							from transformers.models.mobilevit.modeling_tf_mobilevit import TF_MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
							from PIL import Image

							from transformers import MobileViTImageProcessor

class   A__							(						_a     ):




						def 							__UpperCamelCase (				self    :					int     )							->   List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.config_class(**self.inputs_dict     )
										self.parent.assertTrue(hasattr(_UpperCAmelCase ,			'''hidden_sizes'''     )     )
										self.parent.assertTrue(hasattr(_UpperCAmelCase ,			'''neck_hidden_sizes'''     )     )
										self.parent.assertTrue(hasattr(_UpperCAmelCase ,			'''num_attention_heads'''     )     )

class   A__							:




						def __init__(				self    :					Optional[int] ,			_a    :					str ,			_a    :					List[Any]=13 ,			_a    :					List[str]=32 ,			_a    :					int=2 ,			_a    :					Tuple=3 ,			_a    :					List[str]=640 ,			_a    :					List[str]=4 ,			_a    :					List[str]="silu" ,			_a    :					Dict=3 ,			_a    :					Any=32 ,			_a    :					List[Any]=0.1 ,			_a    :					int=0.1 ,			_a    :					Union[str, Any]=0.1 ,			_a    :					List[Any]=0.02 ,			_a    :					str=True ,			_a    :					Tuple=True ,			_a    :					Optional[Any]=10 ,			_a    :					List[str]=None ,			)							->   Any:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =parent
										_SCREAMING_SNAKE_CASE	     =batch_size
										_SCREAMING_SNAKE_CASE	     =image_size
										_SCREAMING_SNAKE_CASE	     =patch_size
										_SCREAMING_SNAKE_CASE	     =num_channels
										_SCREAMING_SNAKE_CASE	     =last_hidden_size
										_SCREAMING_SNAKE_CASE	     =num_attention_heads
										_SCREAMING_SNAKE_CASE	     =hidden_act
										_SCREAMING_SNAKE_CASE	     =conv_kernel_size
										_SCREAMING_SNAKE_CASE	     =output_stride
										_SCREAMING_SNAKE_CASE	     =hidden_dropout_prob
										_SCREAMING_SNAKE_CASE	     =attention_probs_dropout_prob
										_SCREAMING_SNAKE_CASE	     =classifier_dropout_prob
										_SCREAMING_SNAKE_CASE	     =use_labels
										_SCREAMING_SNAKE_CASE	     =is_training
										_SCREAMING_SNAKE_CASE	     =num_labels
										_SCREAMING_SNAKE_CASE	     =initializer_range
										_SCREAMING_SNAKE_CASE	     =scope




						def 							__UpperCamelCase (				self    :					List[Any]     )							->   Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]     )

										_SCREAMING_SNAKE_CASE	     =None
										_SCREAMING_SNAKE_CASE	     =None
										if self.use_labels:
														_SCREAMING_SNAKE_CASE	     =ids_tensor([self.batch_size] ,			self.num_labels     )
														_SCREAMING_SNAKE_CASE	     =ids_tensor([self.batch_size, self.image_size, self.image_size] ,			self.num_labels     )

										_SCREAMING_SNAKE_CASE	     =self.get_config()

										return config, pixel_values, labels, pixel_labels




						def 							__UpperCamelCase (				self    :					List[str]     )							->   Any:


										"""simple docstring"""





										return MobileViTConfig(
										    image_size=self.image_size ,			patch_size=self.patch_size ,			num_channels=self.num_channels ,			num_attention_heads=self.num_attention_heads ,			hidden_act=self.hidden_act ,			conv_kernel_size=self.conv_kernel_size ,			output_stride=self.output_stride ,			hidden_dropout_prob=self.hidden_dropout_prob ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob ,			classifier_dropout_prob=self.classifier_dropout_prob ,			initializer_range=self.initializer_range ,			)




						def 							__UpperCamelCase (				self    :					Dict ,			_a    :					Optional[int] ,			_a    :					str ,			_a    :					int ,			_a    :					Dict     )							->   Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =TFMobileViTModel(config=_UpperCAmelCase     )
										_SCREAMING_SNAKE_CASE	     =model(_UpperCAmelCase ,			training=_UpperCAmelCase     )
										_SCREAMING_SNAKE_CASE	     =self.image_size // self.output_stride
										self.parent.assertEqual(
										    result.last_hidden_state.shape ,			(self.batch_size, self.last_hidden_size, expected_height, expected_width)     )




						def 							__UpperCamelCase (				self    :					Optional[Any] ,			_a    :					Any ,			_a    :					Optional[Any] ,			_a    :					Tuple ,			_a    :					Union[str, Any]     )							->   Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.num_labels
										_SCREAMING_SNAKE_CASE	     =TFMobileViTForImageClassification(_UpperCAmelCase     )
										_SCREAMING_SNAKE_CASE	     =model(_UpperCAmelCase ,			labels=_UpperCAmelCase ,			training=_UpperCAmelCase     )
										self.parent.assertEqual(result.logits.shape ,			(self.batch_size, self.num_labels)     )




						def 							__UpperCamelCase (				self    :					List[Any] ,			_a    :					Optional[Any] ,			_a    :					int ,			_a    :					Dict ,			_a    :					Optional[Any]     )							->   Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.num_labels
										_SCREAMING_SNAKE_CASE	     =TFMobileViTForSemanticSegmentation(_UpperCAmelCase     )
										_SCREAMING_SNAKE_CASE	     =self.image_size // self.output_stride

										_SCREAMING_SNAKE_CASE	     =model(_UpperCAmelCase ,			training=_UpperCAmelCase     )
										self.parent.assertEqual(
										    result.logits.shape ,			(self.batch_size, self.num_labels, expected_height, expected_width)     )

										_SCREAMING_SNAKE_CASE	     =model(_UpperCAmelCase ,			labels=_UpperCAmelCase ,			training=_UpperCAmelCase     )
										self.parent.assertEqual(
										    result.logits.shape ,			(self.batch_size, self.num_labels, expected_height, expected_width)     )




						def 							__UpperCamelCase (				self    :					Optional[Any]     )							->   List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.prepare_config_and_inputs()
										_SCREAMING_SNAKE_CASE	     =config_and_inputs
										_SCREAMING_SNAKE_CASE	     ={"""pixel_values""": pixel_values}
										return config, inputs_dict

@require_tf
class   A__							(						_a      ,   _a      ,   unittest.TestCase     ):
						UpperCAmelCase									=  (
						    (TFMobileViTModel, TFMobileViTForImageClassification, TFMobileViTForSemanticSegmentation)
						    if is_tf_available()
						    else ()
						)
						UpperCAmelCase									=  (
						    {"feature-extraction": TFMobileViTModel, "image-classification": TFMobileViTForImageClassification}
						    if is_tf_available()
						    else {}
						)

						UpperCAmelCase									=  False
						UpperCAmelCase									=  False
						UpperCAmelCase									=  False
						UpperCAmelCase									=  False
						UpperCAmelCase									=  False




						def 							__UpperCamelCase (				self    :					int     )							->   str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =TFMobileViTModelTester(self     )
										_SCREAMING_SNAKE_CASE	     =TFMobileViTConfigTester(self ,			config_class=_UpperCAmelCase ,			has_text_modality=_UpperCAmelCase     )




						def 							__UpperCamelCase (				self    :					Any     )							->   Tuple:


										"""simple docstring"""





										self.config_tester.run_common_tests()




						@unittest.skip(reason='''MobileViT does not use inputs_embeds'''     )
						def 							__UpperCamelCase (				self    :					str     )							->   Any:


										"""simple docstring"""





										pass




						@unittest.skip(reason='''MobileViT does not support input and output embeddings'''     )
						def 							__UpperCamelCase (				self    :					Optional[Any]     )							->   Optional[int]:


										"""simple docstring"""





										pass




						@unittest.skip(reason='''MobileViT does not output attentions'''     )
						def 							__UpperCamelCase (				self    :					Any     )							->   Optional[Any]:


										"""simple docstring"""





										pass




						def 							__UpperCamelCase (				self    :					Any     )							->   List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
														_SCREAMING_SNAKE_CASE	     =model_class(_UpperCAmelCase     )
														_SCREAMING_SNAKE_CASE	     =inspect.signature(model.call     )
														# signature.parameters is an OrderedDict => so arg_names order is deterministic
														_SCREAMING_SNAKE_CASE	     =[*signature.parameters.keys()]

														_SCREAMING_SNAKE_CASE	     =["""pixel_values"""]
														self.assertListEqual(arg_names[:1] ,			_UpperCAmelCase     )




						def 							__UpperCamelCase (				self    :					Tuple     )							->   Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*_UpperCAmelCase     )




						def 							__UpperCamelCase (				self    :					Any     )							->   int:


										"""simple docstring"""





										def check_hidden_states_output(_a    :					Dict ,			_a    :					Any ,			_a    :					Any     ):
														_SCREAMING_SNAKE_CASE	     =model_class(_UpperCAmelCase     )

														_SCREAMING_SNAKE_CASE	     =model(**self._prepare_for_class(_UpperCAmelCase ,			_UpperCAmelCase     )     )

														_SCREAMING_SNAKE_CASE	     =outputs.hidden_states

														_SCREAMING_SNAKE_CASE	     =5
														self.assertEqual(len(_UpperCAmelCase     ) ,			_UpperCAmelCase     )

														# MobileViT's feature maps are of shape (batch_size, num_channels, height, width)
														# with the width and height being successively divided by 2.
														_SCREAMING_SNAKE_CASE	     =2
														for i in range(len(_UpperCAmelCase     )     ):
																		self.assertListEqual(
																		    list(hidden_states[i].shape[-2:]     ) ,			[self.model_tester.image_size // divisor, self.model_tester.image_size // divisor] ,			)
																		divisor *= 2

														self.assertEqual(self.model_tester.output_stride ,			divisor // 2     )

										_SCREAMING_SNAKE_CASE	     =self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
														_SCREAMING_SNAKE_CASE	     =True
														check_hidden_states_output(_UpperCAmelCase ,			_UpperCAmelCase ,			_UpperCAmelCase     )

														# check that output_hidden_states also work using config
														del inputs_dict["output_hidden_states"]
														_SCREAMING_SNAKE_CASE	     =True

														check_hidden_states_output(_UpperCAmelCase ,			_UpperCAmelCase ,			_UpperCAmelCase     )




						def 							__UpperCamelCase (				self    :					str     )							->   Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_image_classification(*_UpperCAmelCase     )




						def 							__UpperCamelCase (				self    :					Union[str, Any]     )							->   str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_semantic_segmentation(*_UpperCAmelCase     )




						@unittest.skipIf(
						    not is_tf_available() or len(tf.config.list_physical_devices('''GPU'''     )     ) == 0 ,			reason='''TF does not support backprop for grouped convolutions on CPU.''' ,			)
						def 							__UpperCamelCase (				self    :					int     )							->   str:


										"""simple docstring"""





										super().test_dataset_conversion()




						def 							__UpperCamelCase (				self    :					Tuple ,			_a    :					Union[str, Any] ,			_a    :					List[str] ,			_a    :					Dict=2E-1 ,			_a    :					List[str]=2E-1     )							->   Optional[int]:


										"""simple docstring"""





										self.assertTrue(np.allclose(_UpperCAmelCase ,			_UpperCAmelCase ,			atol=_UpperCAmelCase ,			rtol=_UpperCAmelCase     )     )




						@unittest.skipIf(
						    not is_tf_available() or len(tf.config.list_physical_devices('''GPU'''     )     ) == 0 ,			reason='''TF does not support backprop for grouped convolutions on CPU.''' ,			)
						@slow
						def 							__UpperCamelCase (				self    :					str     )							->   Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
														# Since `TFMobileViTModel` cannot operate with the default `fit()` method.
														if model_class.__name__ != "TFMobileViTModel":
																		_SCREAMING_SNAKE_CASE	     =model_class(_UpperCAmelCase     )
																		if getattr(_UpperCAmelCase ,			'''hf_compute_loss''' ,			_UpperCAmelCase     ):
																						super().test_keras_fit()




						def 							__UpperCamelCase (				self    :					Dict     )							->   Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.model_tester.prepare_config_and_inputs_for_common()
										for model_class in self.all_model_classes:
														# set an ignore index to correctly test the masked loss used in
														# `TFMobileViTForSemanticSegmentation`.
														if model_class.__name__ != "TFMobileViTForSemanticSegmentation":
																		_SCREAMING_SNAKE_CASE	     =5

														_SCREAMING_SNAKE_CASE	     =model_class(_UpperCAmelCase     )
														if getattr(_UpperCAmelCase ,			'''hf_compute_loss''' ,			_UpperCAmelCase     ):
																		# The number of elements in the loss should be the same as the number of elements in the label
																		_SCREAMING_SNAKE_CASE	     =self._prepare_for_class(inputs_dict.copy() ,			_UpperCAmelCase ,			return_labels=_UpperCAmelCase     )
																		_SCREAMING_SNAKE_CASE	     =prepared_for_class[
																		    sorted(prepared_for_class.keys() - inputs_dict.keys() ,			reverse=_UpperCAmelCase     )[0]
																		]
																		_SCREAMING_SNAKE_CASE	     =added_label.shape.as_list()[:1]

																		# Test that model correctly compute the loss with kwargs
																		_SCREAMING_SNAKE_CASE	     =self._prepare_for_class(inputs_dict.copy() ,			_UpperCAmelCase ,			return_labels=_UpperCAmelCase     )
																		_SCREAMING_SNAKE_CASE	     ={"""input_ids""", """pixel_values""", """input_features"""}
																		_SCREAMING_SNAKE_CASE	     =possible_input_names.intersection(set(_UpperCAmelCase     )     ).pop()
																		_SCREAMING_SNAKE_CASE	     =prepared_for_class.pop(_UpperCAmelCase     )

																		_SCREAMING_SNAKE_CASE	     =model(_UpperCAmelCase ,			**_UpperCAmelCase     )[0]
																		self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]     )

																		# Test that model correctly compute the loss when we mask some positions
																		_SCREAMING_SNAKE_CASE	     =self._prepare_for_class(inputs_dict.copy() ,			_UpperCAmelCase ,			return_labels=_UpperCAmelCase     )
																		_SCREAMING_SNAKE_CASE	     ={"""input_ids""", """pixel_values""", """input_features"""}
																		_SCREAMING_SNAKE_CASE	     =possible_input_names.intersection(set(_UpperCAmelCase     )     ).pop()
																		_SCREAMING_SNAKE_CASE	     =prepared_for_class.pop(_UpperCAmelCase     )
																		if "labels" in prepared_for_class:
																						_SCREAMING_SNAKE_CASE	     =prepared_for_class["""labels"""].numpy()
																						if len(labels.shape     ) > 1 and labels.shape[1] != 1:
																										# labels[0] = -100
																										_SCREAMING_SNAKE_CASE	     =tf.convert_to_tensor(_UpperCAmelCase     )
																										_SCREAMING_SNAKE_CASE	     =model(_UpperCAmelCase ,			**_UpperCAmelCase     )[0]
																										self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]     )
																										self.assertTrue(not np.any(np.isnan(loss.numpy()     )     )     )

                # Test that model correctly compute the loss with a dict
																		_SCREAMING_SNAKE_CASE	     =self._prepare_for_class(inputs_dict.copy() ,			_UpperCAmelCase ,			return_labels=_UpperCAmelCase     )
																		_SCREAMING_SNAKE_CASE	     =model(_UpperCAmelCase     )[0]
																		self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]     )

																		# Test that model correctly compute the loss with a tuple
																		_SCREAMING_SNAKE_CASE	     =self._prepare_for_class(inputs_dict.copy() ,			_UpperCAmelCase ,			return_labels=_UpperCAmelCase     )

																		# Get keys that were added with the _prepare_for_class function
																		_SCREAMING_SNAKE_CASE	     =prepared_for_class.keys() - inputs_dict.keys()
																		_SCREAMING_SNAKE_CASE	     =inspect.signature(model.call     ).parameters
																		_SCREAMING_SNAKE_CASE	     =list(signature.keys()     )

																		# Create a dictionary holding the location of the tensors in the tuple
																		_SCREAMING_SNAKE_CASE	     ={0: input_name}
																		for label_key in label_keys:
																						_SCREAMING_SNAKE_CASE	     =signature_names.index(_UpperCAmelCase     )
																						_SCREAMING_SNAKE_CASE	     =label_key
																		_SCREAMING_SNAKE_CASE	     =sorted(tuple_index_mapping.items()     )
																		# Initialize a list with their default values, update the values and convert to a tuple
																		_SCREAMING_SNAKE_CASE	     =[]

																		for name in signature_names:
																						if name != "kwargs":
																										list_input.append(signature[name].default     )

																		for index, value in sorted_tuple_index_mapping:
																						_SCREAMING_SNAKE_CASE	     =prepared_for_class[value]

																		_SCREAMING_SNAKE_CASE	     =tuple(_UpperCAmelCase     )

																		# Send to model
																		_SCREAMING_SNAKE_CASE	     =model(tuple_input[:-1]     )[0]

																		self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]     )




						@slow
						def 							__UpperCamelCase (				self    :					Union[str, Any]     )							->   int:


										"""simple docstring"""





										for model_name in TF_MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
														_SCREAMING_SNAKE_CASE	     =TFMobileViTModel.from_pretrained(_UpperCAmelCase     )
														self.assertIsNotNone(_UpperCAmelCase     )







def   lowerCamelCase(      ):
				_SCREAMING_SNAKE_CASE	     =Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png''')
				return image


@require_tf
class   A__							(						unittest.TestCase     ):




						@slow
						def 							__UpperCamelCase (				self    :					Any     )							->   Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =TFMobileViTForImageClassification.from_pretrained('''apple/mobilevit-xx-small'''     )

										_SCREAMING_SNAKE_CASE	     =MobileViTImageProcessor.from_pretrained('''apple/mobilevit-xx-small'''     )
										_SCREAMING_SNAKE_CASE	     =prepare_img()
										_SCREAMING_SNAKE_CASE	     =image_processor(images=_UpperCAmelCase ,			return_tensors='''tf'''     )

										# forward pass
										_SCREAMING_SNAKE_CASE	     =model(**_UpperCAmelCase ,			training=_UpperCAmelCase     )

										# verify the logits
										_SCREAMING_SNAKE_CASE	     =tf.TensorShape((1, 1000)     )
										self.assertEqual(outputs.logits.shape ,			_UpperCAmelCase     )

										_SCREAMING_SNAKE_CASE	     =tf.constant([-1.93_64, -1.23_27, -0.46_53]     )

										tf.debugging.assert_near(outputs.logits[0, :3] ,			_UpperCAmelCase ,			atol=1E-4 ,			rtol=1E-04     )




						@slow
						def 							__UpperCamelCase (				self    :					Union[str, Any]     )							->   Optional[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =TFMobileViTForSemanticSegmentation.from_pretrained('''apple/deeplabv3-mobilevit-xx-small'''     )

										_SCREAMING_SNAKE_CASE	     =MobileViTImageProcessor.from_pretrained('''apple/deeplabv3-mobilevit-xx-small'''     )

										_SCREAMING_SNAKE_CASE	     =prepare_img()
										_SCREAMING_SNAKE_CASE	     =image_processor(images=_UpperCAmelCase ,			return_tensors='''tf'''     )

										# forward pass
										_SCREAMING_SNAKE_CASE	     =model(inputs.pixel_values ,			training=_UpperCAmelCase     )
										_SCREAMING_SNAKE_CASE	     =outputs.logits

										# verify the logits
										_SCREAMING_SNAKE_CASE	     =tf.TensorShape((1, 21, 32, 32)     )
										self.assertEqual(logits.shape ,			_UpperCAmelCase     )

										_SCREAMING_SNAKE_CASE	     =tf.constant(
										    [
										        [[6.97_13, 6.97_86, 7.24_22], [7.28_93, 7.28_25, 7.44_46], [7.65_80, 7.87_97, 7.94_20]],
										        [[-10.68_69, -10.32_50, -10.34_71], [-10.42_28, -9.98_68, -9.71_32], [-11.04_05, -11.02_21, -10.73_18]],
										        [[-3.30_89, -2.85_39, -2.67_40], [-3.27_06, -2.56_21, -2.51_08], [-3.25_34, -2.66_15, -2.66_51]],
										    ]     )

										tf.debugging.assert_near(logits[0, :3, :3, :3] ,			_UpperCAmelCase ,			rtol=1E-4 ,			atol=1E-4     )





import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


_lowerCamelCase					:     Optional[Any]					  =  logging.get_logger(__name__)

_lowerCamelCase					:     Dict					  =  {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_lowerCamelCase					:     List[str]					  =  {
    "vocab_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
    },
    "tokenizer_file": {
        "allenai/longformer-base-4096": (
            "https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json"
        ),
    },
}

_lowerCamelCase					:     Dict					  =  {
    "allenai/longformer-base-4096": 4_0_9_6,
    "allenai/longformer-large-4096": 4_0_9_6,
    "allenai/longformer-large-4096-finetuned-triviaqa": 4_0_9_6,
    "allenai/longformer-base-4096-extra.pos.embd.only": 4_0_9_6,
    "allenai/longformer-large-4096-extra.pos.embd.only": 4_0_9_6,
}
class 				__snake_case     (_a						):
				lowerCAmelCase__			=   VOCAB_FILES_NAMES
				lowerCAmelCase__			=   PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__			=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__			=   ["input_ids", "attention_mask"]
				lowerCAmelCase__			=   LongformerTokenizer



				def __init__(					self					:						Optional[Any]						,		_UpperCAmelCase					:						Tuple=None						,		_UpperCAmelCase					:						List[str]=None						,		_UpperCAmelCase					:						List[str]=None						,		_UpperCAmelCase					:						Union[str, Any]="replace"						,		_UpperCAmelCase					:						str="<s>"						,		_UpperCAmelCase					:						str="</s>"						,		_UpperCAmelCase					:						int="</s>"						,		_UpperCAmelCase					:						str="<s>"						,		_UpperCAmelCase					:						Optional[Any]="<unk>"						,		_UpperCAmelCase					:						Optional[Any]="<pad>"						,		_UpperCAmelCase					:						Any="<mask>"						,		_UpperCAmelCase					:						Optional[int]=False						,		_UpperCAmelCase					:						List[str]=True						,		**_UpperCAmelCase					:						Optional[int]						,		)     ->			str:





								'''simple docstring'''




								super().__init__(
								    _UpperCAmelCase						,		_UpperCAmelCase						,		tokenizer_file=_UpperCAmelCase						,		errors=_UpperCAmelCase						,		bos_token=_UpperCAmelCase						,		eos_token=_UpperCAmelCase						,		sep_token=_UpperCAmelCase						,		cls_token=_UpperCAmelCase						,		unk_token=_UpperCAmelCase						,		pad_token=_UpperCAmelCase						,		mask_token=_UpperCAmelCase						,		add_prefix_space=_UpperCAmelCase						,		trim_offsets=_UpperCAmelCase						,		**_UpperCAmelCase						,		)

								_lowerCAmelCase			:			Tuple        =					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
								if pre_tok_state.get("""add_prefix_space"""						,		_UpperCAmelCase					) != add_prefix_space:
												_lowerCAmelCase			:			str        =					getattr(_UpperCAmelCase						,		pre_tok_state.pop("""type"""					)					)
												_lowerCAmelCase			:			Dict        =					add_prefix_space
												_lowerCAmelCase			:			Union[str, Any]        =					pre_tok_class(**_UpperCAmelCase					)

								_lowerCAmelCase			:			List[Any]        =					add_prefix_space

								_lowerCAmelCase			:			List[str]        =					"""post_processor"""
								_lowerCAmelCase			:			str        =					getattr(self.backend_tokenizer						,		_UpperCAmelCase						,		_UpperCAmelCase					)
								if tokenizer_component_instance:
												_lowerCAmelCase			:			Any        =					json.loads(tokenizer_component_instance.__getstate__()					)

												# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
												if "sep" in state:
																_lowerCAmelCase			:			str        =					tuple(state["""sep"""]					)
												if "cls" in state:
																_lowerCAmelCase			:			int        =					tuple(state["""cls"""]					)

												_lowerCAmelCase			:			Any        =					False

												if state.get("""add_prefix_space"""						,		_UpperCAmelCase					) != add_prefix_space:
																_lowerCAmelCase			:			int        =					add_prefix_space
																_lowerCAmelCase			:			List[str]        =					True

												if state.get("""trim_offsets"""						,		_UpperCAmelCase					) != trim_offsets:
																_lowerCAmelCase			:			List[str]        =					trim_offsets
																_lowerCAmelCase			:			List[str]        =					True

												if changes_to_apply:
																_lowerCAmelCase			:			List[str]        =					getattr(_UpperCAmelCase						,		state.pop("""type"""					)					)
																_lowerCAmelCase			:			Any        =					component_class(**_UpperCAmelCase					)
																setattr(self.backend_tokenizer						,		_UpperCAmelCase						,		_UpperCAmelCase					)



				@property
				def 					SCREAMING_SNAKE_CASE				(					self					:						List[Any]					)     ->			str:





								'''simple docstring'''




								if self._mask_token is None:
												if self.verbose:
																logger.error("""Using mask_token, but it is not set yet."""					)
												return None
								return str(self._mask_token					)



				@mask_token.setter
				def 					SCREAMING_SNAKE_CASE				(					self					:						Any						,		_UpperCAmelCase					:						Any					)     ->			Tuple:





								'''simple docstring'''




								_lowerCAmelCase			:			Tuple        =					AddedToken(_UpperCAmelCase						,		lstrip=_UpperCAmelCase						,		rstrip=_UpperCAmelCase					) if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					) else value
								_lowerCAmelCase			:			List[Any]        =					value



				def 					SCREAMING_SNAKE_CASE				(					self					:						int						,		*_UpperCAmelCase					:						str						,		**_UpperCAmelCase					:						Union[str, Any]					)     ->			BatchEncoding:





								'''simple docstring'''




								_lowerCAmelCase			:			Dict        =					kwargs.get("""is_split_into_words"""						,		_UpperCAmelCase					)
								assert self.add_prefix_space or not is_split_into_words, (
								    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*_UpperCAmelCase						,		**_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						str						,		*_UpperCAmelCase					:						Any						,		**_UpperCAmelCase					:						Optional[Any]					)     ->			BatchEncoding:





								'''simple docstring'''




								_lowerCAmelCase			:			Any        =					kwargs.get("""is_split_into_words"""						,		_UpperCAmelCase					)

								assert self.add_prefix_space or not is_split_into_words, (
								    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*_UpperCAmelCase						,		**_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Dict						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						Optional[str] = None					)     ->			Tuple[str]:





								'''simple docstring'''




								_lowerCAmelCase			:			int        =					self._tokenizer.model.save(_UpperCAmelCase						,		name=_UpperCAmelCase					)
								return tuple(_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Any						,		_UpperCAmelCase					:						Any						,		_UpperCAmelCase					:						str=None					)     ->			List[Any]:





								'''simple docstring'''




								_lowerCAmelCase			:			Any        =					[self.bos_token_id] + token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
												return output

								return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[Any]						,		_UpperCAmelCase					:						List[int]						,		_UpperCAmelCase					:						Optional[List[int]] = None					)     ->			List[int]:





								'''simple docstring'''




								_lowerCAmelCase			:			Dict        =					[self.sep_token_id]
								_lowerCAmelCase			:			Optional[Any]        =					[self.cls_token_id]

								if token_ids_a is None:
												return len(cls + token_ids_a + sep					) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]

