





'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


lowerCAmelCase_   :   int       =					logging.get_logger(__name__)


lowerCAmelCase_   :   Optional[Any]       =					{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

# See all MVP models at https://huggingface.co/models?filter=mvp
lowerCAmelCase_   :   List[str]       =					{
    "vocab_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json",
    },
    "added_tokens.json": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json",
    },
    "merges_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json",
    },
}

lowerCAmelCase_   :   Any       =					{
    "RUCAIBox/mvp": 1024,
}






class       UpperCamelCase__  (		__lowerCAmelCase     ):
	lowerCAmelCase__					:			Any								=						VOCAB_FILES_NAMES
	lowerCAmelCase__					:			Any								=						PRETRAINED_VOCAB_FILES_MAP
	lowerCAmelCase__					:			Dict								=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCAmelCase__					:			str								=						["input_ids", "attention_mask"]
	lowerCAmelCase__					:			Dict								=						MvpTokenizer







	def __init__(				self			:   str , lowerCamelCase			:   Tuple=None , lowerCamelCase			:   Dict=None , lowerCamelCase			:   int=None , lowerCamelCase			:   Dict="replace" , lowerCamelCase			:   Optional[int]="<s>" , lowerCamelCase			:   Any="</s>" , lowerCamelCase			:   List[str]="</s>" , lowerCamelCase			:   List[str]="<s>" , lowerCamelCase			:   List[str]="<unk>" , lowerCamelCase			:   Any="<pad>" , lowerCamelCase			:   Dict="<mask>" , lowerCamelCase			:   str=False , lowerCamelCase			:   Any=True , **lowerCamelCase			:   Union[str, Any] , ):




				'''simple docstring'''
				super().__init__(
				    lowerCamelCase , lowerCamelCase , tokenizer_file=lowerCamelCase , errors=lowerCamelCase , bos_token=lowerCamelCase , eos_token=lowerCamelCase , sep_token=lowerCamelCase , cls_token=lowerCamelCase , unk_token=lowerCamelCase , pad_token=lowerCamelCase , mask_token=lowerCamelCase , add_prefix_space=lowerCamelCase , trim_offsets=lowerCamelCase , **lowerCamelCase , )

				a__     = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
				if pre_tok_state.get("add_prefix_space" , lowerCamelCase		) != add_prefix_space:
							a__     = getattr(lowerCamelCase , pre_tok_state.pop("type"		)		)
							a__     = add_prefix_space
							a__     = pre_tok_class(**lowerCamelCase		)

				a__     = add_prefix_space

				# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
				a__     = "post_processor"
				a__     = getattr(self.backend_tokenizer , lowerCamelCase , lowerCamelCase		)
				if tokenizer_component_instance:
							a__     = json.loads(tokenizer_component_instance.__getstate__()		)

							# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
							if "sep" in state:
										a__     = tuple(state["sep"]		)
							if "cls" in state:
										a__     = tuple(state["cls"]		)

							a__     = False

							if state.get("add_prefix_space" , lowerCamelCase		) != add_prefix_space:
										a__     = add_prefix_space
										a__     = True

							if state.get("trim_offsets" , lowerCamelCase		) != trim_offsets:
										a__     = trim_offsets
										a__     = True

							if changes_to_apply:
										a__     = getattr(lowerCamelCase , state.pop("type"		)		)
										a__     = component_class(**lowerCamelCase		)
										setattr(self.backend_tokenizer , lowerCamelCase , lowerCamelCase		)







	@property
	def 						__a   (				self			:   Union[str, Any]		):




				'''simple docstring'''
				if self._mask_token is None:
							if self.verbose:
										logger.error("Using mask_token, but it is not set yet."		)
							return None
				return str(self._mask_token		)







	@mask_token.setter
	def 						__a   (				self			:   str , lowerCamelCase			:   Optional[Any]		):




				'''simple docstring'''
				a__     = AddedToken(lowerCamelCase , lstrip=lowerCamelCase , rstrip=lowerCamelCase		) if isinstance(lowerCamelCase , lowerCamelCase		) else value
				a__     = value







	def 						__a   (				self			:   Optional[int] , *lowerCamelCase			:   List[str] , **lowerCamelCase			:   int		):




				'''simple docstring'''
				a__     = kwargs.get("is_split_into_words" , lowerCamelCase		)

				if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
							    "to use it with pretokenized inputs."		)

				return super()._batch_encode_plus(*lowerCamelCase , **lowerCamelCase		)







	def 						__a   (				self			:   Union[str, Any] , *lowerCamelCase			:   List[Any] , **lowerCamelCase			:   Optional[Any]		):




				'''simple docstring'''
				a__     = kwargs.get("is_split_into_words" , lowerCamelCase		)

				if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
							    "to use it with pretokenized inputs."		)

				return super()._encode_plus(*lowerCamelCase , **lowerCamelCase		)







	def 						__a   (				self			:   int , lowerCamelCase			:   str , lowerCamelCase			:   Optional[str] = None		):




				'''simple docstring'''
				a__     = self._tokenizer.model.save(lowerCamelCase , name=lowerCamelCase		)
				return tuple(lowerCamelCase		)







	def 						__a   (				self			:   Optional[int] , lowerCamelCase			:   Union[str, Any] , lowerCamelCase			:   List[str]=None		):




				'''simple docstring'''
				a__     = [self.bos_token_id] + token_ids_a + [self.eos_token_id]
				if token_ids_a is None:
							return output

				return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







	def 						__a   (				self			:   List[Any] , lowerCamelCase			:   List[int] , lowerCamelCase			:   Optional[List[int]] = None		):




				'''simple docstring'''
				a__     = [self.sep_token_id]
				a__     = [self.cls_token_id]

				if token_ids_a is None:
							return len(cls + token_ids_a + sep		) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]










'''simple docstring'''


# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from ...utils.dataclasses import (
    ComputeEnvironment,
    DistributedType,
    DynamoBackend,
    PrecisionType,
    SageMakerDistributedType,
)
from ..menu import BulletMenu


lowerCAmelCase_   :   List[Any]       =					[
    "EAGER",
    "AOT_EAGER",
    "INDUCTOR",
    "NVFUSER",
    "AOT_NVFUSER",
    "AOT_CUDAGRAPHS",
    "OFI",
    "FX2TRT",
    "ONNXRT",
    "IPEX",
]







def    _lowerCamelCase  (__lowerCamelCase  :     Tuple					,  __lowerCamelCase  :     Tuple=None					,  __lowerCamelCase  :     Tuple=None					,  __lowerCamelCase  :     List[Any]=None     )   ->    Any:
			a__     = True
			while ask_again:
						a__     = input(__lowerCamelCase     )
						try:
									if default is not None and len(__lowerCamelCase     ) == 0:
												return default
									return convert_value(__lowerCamelCase     ) if convert_value is not None else result
						except Exception:
									if error_message is not None:
												print(__lowerCamelCase     )







def    _lowerCamelCase  (__lowerCamelCase  :     Dict					,  __lowerCamelCase  :     List[Any]=[]					,  __lowerCamelCase  :     str=None					,  __lowerCamelCase  :     List[Any]=0     )   ->    List[Any]:
			a__     = BulletMenu(__lowerCamelCase					,  __lowerCamelCase     )
			a__     = menu.run(default_choice=__lowerCamelCase     )
			return convert_value(__lowerCamelCase     ) if convert_value is not None else result







def    _lowerCamelCase  (__lowerCamelCase  :     Tuple     )   ->    Optional[Any]:
			a__     = int(__lowerCamelCase     )
			return ComputeEnvironment(["LOCAL_MACHINE", "AMAZON_SAGEMAKER"][value]     )







def    _lowerCamelCase  (__lowerCamelCase  :     List[str]     )   ->    Optional[int]:
			a__     = int(__lowerCamelCase     )
			return DistributedType(["NO", "MULTI_CPU", "MULTI_XPU", "MULTI_GPU", "MULTI_NPU", "TPU"][value]     )







def    _lowerCamelCase  (__lowerCamelCase  :     List[Any]     )   ->    Optional[Any]:
			a__     = int(__lowerCamelCase     )
			return DynamoBackend(DYNAMO_BACKENDS[value]     ).value







def    _lowerCamelCase  (__lowerCamelCase  :     str     )   ->    str:
			a__     = int(__lowerCamelCase     )
			return PrecisionType(["no", "fp16", "bf16", "fp8"][value]     )







def    _lowerCamelCase  (__lowerCamelCase  :     int     )   ->    Optional[Any]:
			a__     = int(__lowerCamelCase     )
			return SageMakerDistributedType(["NO", "DATA_PARALLEL", "MODEL_PARALLEL"][value]     )







def    _lowerCamelCase  (__lowerCamelCase  :     Any     )   ->    List[Any]:
			return {"yes": True, "no": False}[value.lower()]






class       UpperCamelCase__  (		argparse.RawDescriptionHelpFormatter     ):







	def 						__a   (				self			:   Dict , lowerCamelCase			:   int , lowerCamelCase			:   Optional[Any] , lowerCamelCase			:   List[Any] , lowerCamelCase			:   Dict		):




				'''simple docstring'''
				a__     = super()._format_usage(lowerCamelCase , lowerCamelCase , lowerCamelCase , lowerCamelCase		)
				a__     = usage.replace("<command> [<args>] " , ""		)
				return usage



