


"""simple docstring"""






from __future__ import annotations





def 				A_       ( snake_case__       ,					snake_case__ = None       ,					snake_case__ = None		)					->  None:
			if start is None:
						_UpperCamelCase					:Tuple					     =	0

			if end is None:
						_UpperCamelCase					:List[str]					     =	len(snake_case__		) - 1

			if start >= end:
						return

			_UpperCamelCase					:Union[str, Any]					     =	(start + end) // 2

			slowsort(snake_case__       ,					snake_case__       ,					snake_case__		)
			slowsort(snake_case__       ,					mid + 1       ,					snake_case__		)

			if sequence[end] < sequence[mid]:
						_UpperCamelCase     ,      _UpperCamelCase					:Optional[Any]					     =	sequence[mid], sequence[end]

			slowsort(snake_case__       ,					snake_case__       ,					end - 1		)


if __name__ == "__main__":
					from doctest import testmod

					testmod()





"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


UpperCamelCase__				:List[Any]			   =	logging.get_logger(__name__)


UpperCamelCase__				:Tuple			   =	{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
UpperCamelCase__				:List[str]			   =	{
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json""",
    },
}

UpperCamelCase__				:List[Any]			   =	{
    """RUCAIBox/mvp""": 1_024,
}






class      A(  lowerCamelCase__		):

		"""simple docstring"""





		A    =							VOCAB_FILES_NAMES
		A    =							PRETRAINED_VOCAB_FILES_MAP
		A    =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		A    =							["input_ids", "attention_mask"]
		A    =							MvpTokenizer
		def __init__( self					,				SCREAMING_SNAKE_CASE__=None					,				SCREAMING_SNAKE_CASE__=None					,				SCREAMING_SNAKE_CASE__=None					,				SCREAMING_SNAKE_CASE__="replace"					,				SCREAMING_SNAKE_CASE__="<s>"					,				SCREAMING_SNAKE_CASE__="</s>"					,				SCREAMING_SNAKE_CASE__="</s>"					,				SCREAMING_SNAKE_CASE__="<s>"					,				SCREAMING_SNAKE_CASE__="<unk>"					,				SCREAMING_SNAKE_CASE__="<pad>"					,				SCREAMING_SNAKE_CASE__="<mask>"					,				SCREAMING_SNAKE_CASE__=False					,				SCREAMING_SNAKE_CASE__=True					,				**SCREAMING_SNAKE_CASE__					,				)      ->	List[str]:

					"""simple docstring"""



					super().__init__(
					    SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__					,				tokenizer_file=SCREAMING_SNAKE_CASE__					,				errors=SCREAMING_SNAKE_CASE__					,				bos_token=SCREAMING_SNAKE_CASE__					,				eos_token=SCREAMING_SNAKE_CASE__					,				sep_token=SCREAMING_SNAKE_CASE__					,				cls_token=SCREAMING_SNAKE_CASE__					,				unk_token=SCREAMING_SNAKE_CASE__					,				pad_token=SCREAMING_SNAKE_CASE__					,				mask_token=SCREAMING_SNAKE_CASE__					,				add_prefix_space=SCREAMING_SNAKE_CASE__					,				trim_offsets=SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__					,				)

					_UpperCamelCase					:int					     =	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
					if pre_tok_state.get('''add_prefix_space'''					,				SCREAMING_SNAKE_CASE__   ) != add_prefix_space:
								_UpperCamelCase					:int					     =	getattr(SCREAMING_SNAKE_CASE__					,				pre_tok_state.pop('''type'''   )   )
								_UpperCamelCase					:Optional[int]					     =	add_prefix_space
								_UpperCamelCase					:List[Any]					     =	pre_tok_class(**SCREAMING_SNAKE_CASE__   )

					_UpperCamelCase					:Optional[int]					     =	add_prefix_space

					# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
					_UpperCamelCase					:Any					     =	'''post_processor'''
					_UpperCamelCase					:Optional[Any]					     =	getattr(self.backend_tokenizer					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )
					if tokenizer_component_instance:
								_UpperCamelCase					:Optional[Any]					     =	json.loads(tokenizer_component_instance.__getstate__()   )

								# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
								if "sep" in state:
											_UpperCamelCase					:Optional[Any]					     =	tuple(state['''sep''']   )
								if "cls" in state:
											_UpperCamelCase					:Optional[Any]					     =	tuple(state['''cls''']   )

								_UpperCamelCase					:str					     =	False

								if state.get('''add_prefix_space'''					,				SCREAMING_SNAKE_CASE__   ) != add_prefix_space:
											_UpperCamelCase					:Optional[int]					     =	add_prefix_space
											_UpperCamelCase					:Union[str, Any]					     =	True

								if state.get('''trim_offsets'''					,				SCREAMING_SNAKE_CASE__   ) != trim_offsets:
											_UpperCamelCase					:Union[str, Any]					     =	trim_offsets
											_UpperCamelCase					:Optional[Any]					     =	True

								if changes_to_apply:
											_UpperCamelCase					:Union[str, Any]					     =	getattr(SCREAMING_SNAKE_CASE__					,				state.pop('''type'''   )   )
											_UpperCamelCase					:Optional[Any]					     =	component_class(**SCREAMING_SNAKE_CASE__   )
											setattr(self.backend_tokenizer					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )
		@property
		def   _UpperCamelCase( self   )      ->	str:

					"""simple docstring"""



					if self._mask_token is None:
								if self.verbose:
											logger.error('''Using mask_token, but it is not set yet.'''   )
								return None
					return str(self._mask_token   )
		@mask_token.setter
		def   _UpperCamelCase( self					,				SCREAMING_SNAKE_CASE__   )      ->	int:

					"""simple docstring"""



					_UpperCamelCase					:List[Any]					     =	AddedToken(SCREAMING_SNAKE_CASE__					,				lstrip=SCREAMING_SNAKE_CASE__					,				rstrip=SCREAMING_SNAKE_CASE__   ) if isinstance(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   ) else value
					_UpperCamelCase					:int					     =	value
		def   _UpperCamelCase( self					,				*SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )      ->	BatchEncoding:

					"""simple docstring"""



					_UpperCamelCase					:int					     =	kwargs.get('''is_split_into_words'''					,				SCREAMING_SNAKE_CASE__   )

					if is_split_into_words and not self.add_prefix_space:
								raise ValueError(
								    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
								    '''to use it with pretokenized inputs.'''   )

					return super()._batch_encode_plus(*SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )
		def   _UpperCamelCase( self					,				*SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )      ->	BatchEncoding:

					"""simple docstring"""



					_UpperCamelCase					:Dict					     =	kwargs.get('''is_split_into_words'''					,				SCREAMING_SNAKE_CASE__   )

					if is_split_into_words and not self.add_prefix_space:
								raise ValueError(
								    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
								    '''to use it with pretokenized inputs.'''   )

					return super()._encode_plus(*SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )
		def   _UpperCamelCase( self					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__ = None   )      ->	Tuple[str]:

					"""simple docstring"""



					_UpperCamelCase					:Optional[Any]					     =	self._tokenizer.model.save(SCREAMING_SNAKE_CASE__					,				name=SCREAMING_SNAKE_CASE__   )
					return tuple(SCREAMING_SNAKE_CASE__   )
		def   _UpperCamelCase( self					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__=None   )      ->	str:

					"""simple docstring"""



					_UpperCamelCase					:Optional[int]					     =	[self.bos_token_id] + token_ids_a + [self.eos_token_id]
					if token_ids_a is None:
								return output

					return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





		def   _UpperCamelCase( self					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__ = None   )      ->	List[int]:

					"""simple docstring"""



					_UpperCamelCase					:Any					     =	[self.sep_token_id]
					_UpperCamelCase					:List[str]					     =	[self.cls_token_id]

					if token_ids_a is None:
								return len(cls + token_ids_a + sep   ) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]

