

from typing import List

import numpy as np






def        snake_case__		(    lowerCamelCase_		):
							A      :					Tuple      		=					{key: len(lowerCamelCase_		) for key, value in gen_kwargs.items() if isinstance(lowerCamelCase_     ,				lowerCamelCase_		)}
							if len(set(lists_lengths.values()		)		) > 1:
														raise RuntimeError(
														    (
														        '''Sharding is ambiguous for this dataset: '''
														        + '''we found several data sources lists of different lengths, and we don\'t know over which list we should parallelize:\n'''
														        + '''\n'''.join(F'\t- key {key} has length {length}' for key, length in lists_lengths.items()		)
														        + '''\nTo fix this, check the \'gen_kwargs\' and make sure to use lists only for data sources, '''
														        + '''and use tuples otherwise. In the end there should only be one single list, or several lists with the same length.'''
														    )		)
							A      :					Tuple      		=					max(lists_lengths.values()     ,				default=0		)
							return max(1     ,				lowerCamelCase_		)






def        snake_case__		(    lowerCamelCase_     ,				lowerCamelCase_		):
							A      :					int      		=					[]
							for group_idx in range(lowerCamelCase_		):
														A      :					Optional[int]      		=					num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs))
														if num_shards_to_add == 0:
																					break
														A      :					List[Any]      		=					shards_indices_per_group[-1].stop if shards_indices_per_group else 0
														A      :					Tuple      		=					range(lowerCamelCase_     ,				start + num_shards_to_add		)
														shards_indices_per_group.append(lowerCamelCase_		)
							return shards_indices_per_group






def        snake_case__		(    lowerCamelCase_     ,				lowerCamelCase_		):
							A      :					List[Any]      		=					_number_of_shards_in_gen_kwargs(lowerCamelCase_		)
							if num_shards == 1:
														return [dict(lowerCamelCase_		)]
							else:
														A      :					Optional[Any]      		=					_distribute_shards(num_shards=lowerCamelCase_     ,				max_num_jobs=lowerCamelCase_		)
														return [
														    {
														        key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]]
														        if isinstance(lowerCamelCase_     ,				lowerCamelCase_		)
														        else value
														        for key, value in gen_kwargs.items()
														    }
														    for group_idx in range(len(lowerCamelCase_		)		)
														]






def        snake_case__		(    lowerCamelCase_		):
							return {
							    key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]]
							    if isinstance(gen_kwargs_list[0][key]     ,				lowerCamelCase_		)
							    else gen_kwargs_list[0][key]
							    for key in gen_kwargs_list[0]
							}






def        snake_case__		(    lowerCamelCase_     ,				lowerCamelCase_		):
							A      :					Any      		=					{len(lowerCamelCase_		) for value in gen_kwargs.values() if isinstance(lowerCamelCase_     ,				lowerCamelCase_		)}
							A      :					List[str]      		=					{}
							for size in list_sizes:
														A      :					Any      		=					list(range(lowerCamelCase_		)		)
														rng.shuffle(indices_per_size[size]		)
							# Now let's copy the gen_kwargs and shuffle the lists based on their sizes
							A      :					Optional[int]      		=					dict(lowerCamelCase_		)
							for key, value in shuffled_kwargs.items():
														if isinstance(lowerCamelCase_     ,				lowerCamelCase_		):
																					A      :					Dict      		=					[value[i] for i in indices_per_size[len(lowerCamelCase_		)]]
							return shuffled_kwargs









import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


lowercase			:				Optional[int]   =     logging.get_logger(__name__)

lowercase			:				Optional[Any]   =     {"vocab_file": "vocab.txt"}

lowercase			:				Optional[int]   =     {
    "vocab_file": {
        "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt",
        "YituTech/conv-bert-medium-small": (
            "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"
        ),
        "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt",
    }
}

lowercase			:				Tuple   =     {
    "YituTech/conv-bert-base": 5_12,
    "YituTech/conv-bert-medium-small": 5_12,
    "YituTech/conv-bert-small": 5_12,
}


lowercase			:				Optional[int]   =     {
    "YituTech/conv-bert-base": {"do_lower_case": True},
    "YituTech/conv-bert-medium-small": {"do_lower_case": True},
    "YituTech/conv-bert-small": {"do_lower_case": True},
}







class 					__lowercase  (						_SCREAMING_SNAKE_CASE				):





							"""simple docstring"""

							UpperCAmelCase_						:		int								=	VOCAB_FILES_NAMES
							UpperCAmelCase_						:		Optional[int]								=	PRETRAINED_VOCAB_FILES_MAP
							UpperCAmelCase_						:		List[Any]								=	PRETRAINED_INIT_CONFIGURATION
							UpperCAmelCase_						:		Optional[Any]								=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCAmelCase_						:		Optional[int]								=	ConvBertTokenizer




							def __init__(					self    ,						__UpperCAmelCase=None    ,						__UpperCAmelCase=None    ,						__UpperCAmelCase=True    ,						__UpperCAmelCase="[UNK]"    ,						__UpperCAmelCase="[SEP]"    ,						__UpperCAmelCase="[PAD]"    ,						__UpperCAmelCase="[CLS]"    ,						__UpperCAmelCase="[MASK]"    ,						__UpperCAmelCase=True    ,						__UpperCAmelCase=None    ,						**__UpperCAmelCase    ,						)		->       Optional[int]:
														super().__init__(
														    __UpperCAmelCase    ,						tokenizer_file=__UpperCAmelCase    ,						do_lower_case=__UpperCAmelCase    ,						unk_token=__UpperCAmelCase    ,						sep_token=__UpperCAmelCase    ,						pad_token=__UpperCAmelCase    ,						cls_token=__UpperCAmelCase    ,						mask_token=__UpperCAmelCase    ,						tokenize_chinese_chars=__UpperCAmelCase    ,						strip_accents=__UpperCAmelCase    ,						**__UpperCAmelCase    ,						)

														A      :					Any      		=					json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
														if (
														    normalizer_state.get('''lowercase'''    ,						__UpperCAmelCase						) != do_lower_case
														    or normalizer_state.get('''strip_accents'''    ,						__UpperCAmelCase						) != strip_accents
														    or normalizer_state.get('''handle_chinese_chars'''    ,						__UpperCAmelCase						) != tokenize_chinese_chars
														):
																					A      :					Tuple      		=					getattr(__UpperCAmelCase    ,						normalizer_state.pop('''type'''						)						)
																					A      :					int      		=					do_lower_case
																					A      :					int      		=					strip_accents
																					A      :					Union[str, Any]      		=					tokenize_chinese_chars
																					A      :					str      		=					normalizer_class(**__UpperCAmelCase						)

														A      :					str      		=					do_lower_case




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase=None						)		->       Optional[int]:
														A      :					Dict      		=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase = None						)		->       List[int]:
														A      :					List[Any]      		=					[self.sep_token_id]
														A      :					Optional[Any]      		=					[self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep						) * [0]
														return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase = None						)		->       Tuple[str]:
														A      :					Optional[int]      		=					self._tokenizer.model.save(__UpperCAmelCase    ,						name=__UpperCAmelCase						)
														return tuple(__UpperCAmelCase						)






