


import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


UpperCamelCase														=      logging.get_logger(__name__)


UpperCamelCase														=      {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

# See all MVP models at https://huggingface.co/models?filter=mvp
UpperCamelCase														=      {
    '''vocab_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json''',
    },
    '''added_tokens.json''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json''',
    },
    '''merges_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json''',
    },
}

UpperCamelCase														=      {
    '''RUCAIBox/mvp''': 1024,
}






class     _lowerCamelCase					(   lowerCamelCase_     ):




       """simple docstring"""
       snake_case    =       VOCAB_FILES_NAMES
       snake_case    =       PRETRAINED_VOCAB_FILES_MAP
       snake_case    =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       snake_case    =       ["input_ids", "attention_mask"]
       snake_case    =       MvpTokenizer



       def __init__( self							,			_SCREAMING_SNAKE_CASE=None							,			_SCREAMING_SNAKE_CASE=None							,			_SCREAMING_SNAKE_CASE=None							,			_SCREAMING_SNAKE_CASE="replace"							,			_SCREAMING_SNAKE_CASE="<s>"							,			_SCREAMING_SNAKE_CASE="</s>"							,			_SCREAMING_SNAKE_CASE="</s>"							,			_SCREAMING_SNAKE_CASE="<s>"							,			_SCREAMING_SNAKE_CASE="<unk>"							,			_SCREAMING_SNAKE_CASE="<pad>"							,			_SCREAMING_SNAKE_CASE="<mask>"							,			_SCREAMING_SNAKE_CASE=False							,			_SCREAMING_SNAKE_CASE=True							,			**_SCREAMING_SNAKE_CASE							,			)->List[Any]:




         '''simple docstring'''





         super().__init__(
             snake_case__							,			snake_case__							,			tokenizer_file=snake_case__							,			errors=snake_case__							,			bos_token=snake_case__							,			eos_token=snake_case__							,			sep_token=snake_case__							,			cls_token=snake_case__							,			unk_token=snake_case__							,			pad_token=snake_case__							,			mask_token=snake_case__							,			add_prefix_space=snake_case__							,			trim_offsets=snake_case__							,			**snake_case__							,			)

         A_ :							str							=   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
         if pre_tok_state.get('''add_prefix_space'''							,			snake_case__						) != add_prefix_space:
           A_ :							Tuple							=   getattr(snake_case__							,			pre_tok_state.pop('''type'''						)						)
           A_ :							Tuple							=   add_prefix_space
           A_ :							Union[str, Any]							=   pre_tok_class(**snake_case__						)

         A_ :							int							=   add_prefix_space

         # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
         A_ :							Optional[int]							=   'post_processor'
         A_ :							str							=   getattr(self.backend_tokenizer							,			snake_case__							,			snake_case__						)
         if tokenizer_component_instance:
           A_ :							Tuple							=   json.loads(tokenizer_component_instance.__getstate__()						)

           # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
           if "sep" in state:
             A_ :							List[str]							=   tuple(state['''sep''']						)
           if "cls" in state:
             A_ :							Optional[int]							=   tuple(state['''cls''']						)

           A_ :							List[Any]							=   False

           if state.get('''add_prefix_space'''							,			snake_case__						) != add_prefix_space:
             A_ :							Any							=   add_prefix_space
             A_ :							str							=   True

           if state.get('''trim_offsets'''							,			snake_case__						) != trim_offsets:
             A_ :							Optional[int]							=   trim_offsets
             A_ :							str							=   True

           if changes_to_apply:
             A_ :							Union[str, Any]							=   getattr(snake_case__							,			state.pop('''type'''						)						)
             A_ :							Optional[int]							=   component_class(**snake_case__						)
             setattr(self.backend_tokenizer							,			snake_case__							,			snake_case__						)



       @property
       def 			_snake_case			( self						)->int:




         '''simple docstring'''





         if self._mask_token is None:
           if self.verbose:
             logger.error('''Using mask_token, but it is not set yet.'''						)
           return None
         return str(self._mask_token						)



       @mask_token.setter
       def 			_snake_case			( self							,			_SCREAMING_SNAKE_CASE						)->Dict:




         '''simple docstring'''





         A_ :							Optional[int]							=   AddedToken(snake_case__							,			lstrip=snake_case__							,			rstrip=snake_case__						) if isinstance(snake_case__							,			snake_case__						) else value
         A_ :							Tuple							=   value



       def 			_snake_case			( self							,			*_SCREAMING_SNAKE_CASE							,			**_SCREAMING_SNAKE_CASE						)->Union[str, Any]:




         '''simple docstring'''





         A_ :							Optional[Any]							=   kwargs.get('''is_split_into_words'''							,			snake_case__						)

         if is_split_into_words and not self.add_prefix_space:
           raise ValueError(
               F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
               '''to use it with pretokenized inputs.'''						)

         return super()._batch_encode_plus(*snake_case__							,			**snake_case__						)



       def 			_snake_case			( self							,			*_SCREAMING_SNAKE_CASE							,			**_SCREAMING_SNAKE_CASE						)->Optional[Any]:




         '''simple docstring'''





         A_ :							List[Any]							=   kwargs.get('''is_split_into_words'''							,			snake_case__						)

         if is_split_into_words and not self.add_prefix_space:
           raise ValueError(
               F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
               '''to use it with pretokenized inputs.'''						)

         return super()._encode_plus(*snake_case__							,			**snake_case__						)



       def 			_snake_case			( self							,			_SCREAMING_SNAKE_CASE							,			_SCREAMING_SNAKE_CASE = None						)->Dict:




         '''simple docstring'''





         A_ :							Optional[Any]							=   self._tokenizer.model.save(snake_case__							,			name=snake_case__						)
         return tuple(snake_case__						)



       def 			_snake_case			( self							,			_SCREAMING_SNAKE_CASE							,			_SCREAMING_SNAKE_CASE=None						)->Optional[Any]:




         '''simple docstring'''





         A_ :							int							=   [self.bos_token_id] + token_ids_a + [self.eos_token_id]
         if token_ids_a is None:
           return output

         return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







       def 			_snake_case			( self							,			_SCREAMING_SNAKE_CASE							,			_SCREAMING_SNAKE_CASE = None						)->Dict:




         '''simple docstring'''





         A_ :							str							=   [self.sep_token_id]
         A_ :							Any							=   [self.cls_token_id]

         if token_ids_a is None:
           return len(cls + token_ids_a + sep						) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]










from typing import List, Optional, TypeVar

from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .info import DatasetInfo
from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets
from .splits import NamedSplit
from .utils import logging
from .utils.py_utils import Literal


UpperCamelCase__ :   Optional[Any]						=							logging.get_logger(__name__)


UpperCamelCase__ :   int						=							TypeVar('''DatasetType''', Dataset, IterableDataset)





def        __UpperCAmelCase						(							lowerCamelCase_ :     List[DatasetType]  ,							lowerCamelCase_ :     Optional[List[float]] = None  ,							lowerCamelCase_ :     Optional[int] = None  ,							lowerCamelCase_ :     Optional[DatasetInfo] = None  ,							lowerCamelCase_ :     Optional[NamedSplit] = None  ,							lowerCamelCase_ :     Literal["first_exhausted", "all_exhausted"] = "first_exhausted"  ,							)       ->  DatasetType:







			"""simple docstring"""





			from .arrow_dataset import Dataset
			from .iterable_dataset import IterableDataset

			if not datasets:
						raise ValueError('Unable to interleave an empty list of datasets.'  )
			for i, dataset in enumerate(lowerCamelCase_  ):
						if not isinstance(lowerCamelCase_  ,							(Dataset, IterableDataset)  ):
									if isinstance(lowerCamelCase_  ,							(DatasetDict, IterableDatasetDict)  ):
												if not dataset:
															raise ValueError(
															    F'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} '
															    'is an empty dataset dictionary.'  )
												raise ValueError(
												    F'Dataset at position {i} has at least one split: {list(lowerCamelCase_  )}\n'
												    F'Please pick one to interleave with the other datasets, for example: dataset[\'{next(iter(lowerCamelCase_  )  )}\']'  )
									raise ValueError(
									    F'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(lowerCamelCase_  ).__name__}.'  )
						if i == 0:
									SCREAMING_SNAKE_CASE_  ,    SCREAMING_SNAKE_CASE_   : Dict					  =      (
									    (Dataset, IterableDataset) if isinstance(lowerCamelCase_  ,							lowerCamelCase_  ) else (IterableDataset, Dataset)
									)
						elif not isinstance(lowerCamelCase_  ,							lowerCamelCase_  ):
									raise ValueError(
									    F'Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects.'  )
			if stopping_strategy not in ["first_exhausted", "all_exhausted"]:
						raise ValueError(F'{stopping_strategy} is not supported. Please enter a valid stopping_strategy.'  )
			if dataset_type is Dataset:
						return _interleave_map_style_datasets(
						    lowerCamelCase_  ,							lowerCamelCase_  ,							lowerCamelCase_  ,							info=lowerCamelCase_  ,							split=lowerCamelCase_  ,							stopping_strategy=lowerCamelCase_  )
			else:
						return _interleave_iterable_datasets(
						    lowerCamelCase_  ,							lowerCamelCase_  ,							lowerCamelCase_  ,							info=lowerCamelCase_  ,							split=lowerCamelCase_  ,							stopping_strategy=lowerCamelCase_  )





def        __UpperCAmelCase						(							lowerCamelCase_ :     List[DatasetType]  ,							lowerCamelCase_ :     Optional[DatasetInfo] = None  ,							lowerCamelCase_ :     Optional[NamedSplit] = None  ,							lowerCamelCase_ :     int = 0  ,							)       ->  DatasetType:







			"""simple docstring"""





			if not dsets:
						raise ValueError('Unable to concatenate an empty list of datasets.'  )
			for i, dataset in enumerate(lowerCamelCase_  ):
						if not isinstance(lowerCamelCase_  ,							(Dataset, IterableDataset)  ):
									if isinstance(lowerCamelCase_  ,							(DatasetDict, IterableDatasetDict)  ):
												if not dataset:
															raise ValueError(
															    F'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} '
															    'is an empty dataset dictionary.'  )
												raise ValueError(
												    F'Dataset at position {i} has at least one split: {list(lowerCamelCase_  )}\n'
												    F'Please pick one to interleave with the other datasets, for example: dataset[\'{next(iter(lowerCamelCase_  )  )}\']'  )
									raise ValueError(
									    F'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(lowerCamelCase_  ).__name__}.'  )
						if i == 0:
									SCREAMING_SNAKE_CASE_  ,    SCREAMING_SNAKE_CASE_   : Any					  =      (
									    (Dataset, IterableDataset) if isinstance(lowerCamelCase_  ,							lowerCamelCase_  ) else (IterableDataset, Dataset)
									)
						elif not isinstance(lowerCamelCase_  ,							lowerCamelCase_  ):
									raise ValueError(
									    F'Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects.'  )
			if dataset_type is Dataset:
						return _concatenate_map_style_datasets(lowerCamelCase_  ,							info=lowerCamelCase_  ,							split=lowerCamelCase_  ,							axis=lowerCamelCase_  )
			else:
						return _concatenate_iterable_datasets(lowerCamelCase_  ,							info=lowerCamelCase_  ,							split=lowerCamelCase_  ,							axis=lowerCamelCase_  )



