




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


UpperCamelCase__ :   Union[str, Any]						=							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase__ :   int						=							{
    '''vocab_file''': {
        '''google/electra-small-generator''': (
            '''https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt'''
        ),
        '''google/electra-base-generator''': '''https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt''',
        '''google/electra-large-generator''': (
            '''https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt'''
        ),
        '''google/electra-small-discriminator''': (
            '''https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt'''
        ),
        '''google/electra-base-discriminator''': (
            '''https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt'''
        ),
        '''google/electra-large-discriminator''': (
            '''https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''google/electra-small-generator''': (
            '''https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-base-generator''': (
            '''https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-large-generator''': (
            '''https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-small-discriminator''': (
            '''https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-base-discriminator''': (
            '''https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-large-discriminator''': (
            '''https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase__ :   int						=							{
    '''google/electra-small-generator''': 5_12,
    '''google/electra-base-generator''': 5_12,
    '''google/electra-large-generator''': 5_12,
    '''google/electra-small-discriminator''': 5_12,
    '''google/electra-base-discriminator''': 5_12,
    '''google/electra-large-discriminator''': 5_12,
}

UpperCamelCase__ :   List[Any]						=							{
    '''google/electra-small-generator''': {'''do_lower_case''': True},
    '''google/electra-base-generator''': {'''do_lower_case''': True},
    '''google/electra-large-generator''': {'''do_lower_case''': True},
    '''google/electra-small-discriminator''': {'''do_lower_case''': True},
    '''google/electra-base-discriminator''': {'''do_lower_case''': True},
    '''google/electra-large-discriminator''': {'''do_lower_case''': True},
}







class 				lowerCAmelCase_       (					lowerCamelCase_							):
				__a     :       List[Any]						=     VOCAB_FILES_NAMES
				__a     :       Any						=     PRETRAINED_VOCAB_FILES_MAP
				__a     :       Dict						=     PRETRAINED_INIT_CONFIGURATION
				__a     :       Union[str, Any]						=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__a     :       List[str]						=     ElectraTokenizer






				def __init__(			self      ,snake_case__=None      ,snake_case__=None      ,snake_case__=True      ,snake_case__="[UNK]"      ,snake_case__="[SEP]"      ,snake_case__="[PAD]"      ,snake_case__="[CLS]"      ,snake_case__="[MASK]"      ,snake_case__=True      ,snake_case__=None      ,**snake_case__      ,):
							super().__init__(
							    snake_case__      ,tokenizer_file=snake_case__      ,do_lower_case=snake_case__      ,unk_token=snake_case__      ,sep_token=snake_case__      ,pad_token=snake_case__      ,cls_token=snake_case__      ,mask_token=snake_case__      ,tokenize_chinese_chars=snake_case__      ,strip_accents=snake_case__      ,**snake_case__      ,)

							SCREAMING_SNAKE_CASE_   : Any					  =      json.loads(self.backend_tokenizer.normalizer.__getstate__() )
							if (
							    normalizer_state.get('lowercase'      ,snake_case__ ) != do_lower_case
							    or normalizer_state.get('strip_accents'      ,snake_case__ ) != strip_accents
							    or normalizer_state.get('handle_chinese_chars'      ,snake_case__ ) != tokenize_chinese_chars
							):
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      getattr(snake_case__      ,normalizer_state.pop('type' ) )
										SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      do_lower_case
										SCREAMING_SNAKE_CASE_   : List[str]					  =      strip_accents
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      tokenize_chinese_chars
										SCREAMING_SNAKE_CASE_   : int					  =      normalizer_class(**snake_case__ )

							SCREAMING_SNAKE_CASE_   : List[str]					  =      do_lower_case






				def 		snake_case	(			self      ,snake_case__      ,snake_case__=None ):
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
										output += token_ids_a + [self.sep_token_id]

							return output






				def 		snake_case	(			self      ,snake_case__      ,snake_case__ = None ):
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      [self.sep_token_id]
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      [self.cls_token_id]
							if token_ids_a is None:
										return len(cls + token_ids_a + sep ) * [0]
							return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






				def 		snake_case	(			self      ,snake_case__      ,snake_case__ = None ):
							SCREAMING_SNAKE_CASE_   : int					  =      self._tokenizer.model.save(snake_case__      ,name=snake_case__ )
							return tuple(snake_case__ )









from typing import List, Optional, TypeVar

from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .info import DatasetInfo
from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets
from .splits import NamedSplit
from .utils import logging
from .utils.py_utils import Literal


UpperCamelCase__ :   Optional[Any]						=							logging.get_logger(__name__)


UpperCamelCase__ :   int						=							TypeVar('''DatasetType''', Dataset, IterableDataset)





def        __UpperCAmelCase						(							lowerCamelCase_ :     List[DatasetType]  ,							lowerCamelCase_ :     Optional[List[float]] = None  ,							lowerCamelCase_ :     Optional[int] = None  ,							lowerCamelCase_ :     Optional[DatasetInfo] = None  ,							lowerCamelCase_ :     Optional[NamedSplit] = None  ,							lowerCamelCase_ :     Literal["first_exhausted", "all_exhausted"] = "first_exhausted"  ,							)       ->  DatasetType:







			"""simple docstring"""





			from .arrow_dataset import Dataset
			from .iterable_dataset import IterableDataset

			if not datasets:
						raise ValueError('Unable to interleave an empty list of datasets.'  )
			for i, dataset in enumerate(lowerCamelCase_  ):
						if not isinstance(lowerCamelCase_  ,							(Dataset, IterableDataset)  ):
									if isinstance(lowerCamelCase_  ,							(DatasetDict, IterableDatasetDict)  ):
												if not dataset:
															raise ValueError(
															    F'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} '
															    'is an empty dataset dictionary.'  )
												raise ValueError(
												    F'Dataset at position {i} has at least one split: {list(lowerCamelCase_  )}\n'
												    F'Please pick one to interleave with the other datasets, for example: dataset[\'{next(iter(lowerCamelCase_  )  )}\']'  )
									raise ValueError(
									    F'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(lowerCamelCase_  ).__name__}.'  )
						if i == 0:
									SCREAMING_SNAKE_CASE_  ,    SCREAMING_SNAKE_CASE_   : Dict					  =      (
									    (Dataset, IterableDataset) if isinstance(lowerCamelCase_  ,							lowerCamelCase_  ) else (IterableDataset, Dataset)
									)
						elif not isinstance(lowerCamelCase_  ,							lowerCamelCase_  ):
									raise ValueError(
									    F'Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects.'  )
			if stopping_strategy not in ["first_exhausted", "all_exhausted"]:
						raise ValueError(F'{stopping_strategy} is not supported. Please enter a valid stopping_strategy.'  )
			if dataset_type is Dataset:
						return _interleave_map_style_datasets(
						    lowerCamelCase_  ,							lowerCamelCase_  ,							lowerCamelCase_  ,							info=lowerCamelCase_  ,							split=lowerCamelCase_  ,							stopping_strategy=lowerCamelCase_  )
			else:
						return _interleave_iterable_datasets(
						    lowerCamelCase_  ,							lowerCamelCase_  ,							lowerCamelCase_  ,							info=lowerCamelCase_  ,							split=lowerCamelCase_  ,							stopping_strategy=lowerCamelCase_  )





def        __UpperCAmelCase						(							lowerCamelCase_ :     List[DatasetType]  ,							lowerCamelCase_ :     Optional[DatasetInfo] = None  ,							lowerCamelCase_ :     Optional[NamedSplit] = None  ,							lowerCamelCase_ :     int = 0  ,							)       ->  DatasetType:







			"""simple docstring"""





			if not dsets:
						raise ValueError('Unable to concatenate an empty list of datasets.'  )
			for i, dataset in enumerate(lowerCamelCase_  ):
						if not isinstance(lowerCamelCase_  ,							(Dataset, IterableDataset)  ):
									if isinstance(lowerCamelCase_  ,							(DatasetDict, IterableDatasetDict)  ):
												if not dataset:
															raise ValueError(
															    F'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} '
															    'is an empty dataset dictionary.'  )
												raise ValueError(
												    F'Dataset at position {i} has at least one split: {list(lowerCamelCase_  )}\n'
												    F'Please pick one to interleave with the other datasets, for example: dataset[\'{next(iter(lowerCamelCase_  )  )}\']'  )
									raise ValueError(
									    F'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(lowerCamelCase_  ).__name__}.'  )
						if i == 0:
									SCREAMING_SNAKE_CASE_  ,    SCREAMING_SNAKE_CASE_   : Any					  =      (
									    (Dataset, IterableDataset) if isinstance(lowerCamelCase_  ,							lowerCamelCase_  ) else (IterableDataset, Dataset)
									)
						elif not isinstance(lowerCamelCase_  ,							lowerCamelCase_  ):
									raise ValueError(
									    F'Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects.'  )
			if dataset_type is Dataset:
						return _concatenate_map_style_datasets(lowerCamelCase_  ,							info=lowerCamelCase_  ,							split=lowerCamelCase_  ,							axis=lowerCamelCase_  )
			else:
						return _concatenate_iterable_datasets(lowerCamelCase_  ,							info=lowerCamelCase_  ,							split=lowerCamelCase_  ,							axis=lowerCamelCase_  )



