
"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


__magic_name__			=     logging.get_logger(__name__)

__magic_name__			=     {"vocab_file": "vocab.txt"}

__magic_name__			=     {
    "vocab_file": {
        "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt",
        "YituTech/conv-bert-medium-small": (
            "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"
        ),
        "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt",
    }
}

__magic_name__			=     {
    "YituTech/conv-bert-base": 512,
    "YituTech/conv-bert-medium-small": 512,
    "YituTech/conv-bert-small": 512,
}


__magic_name__			=     {
    "YituTech/conv-bert-base": {"do_lower_case": True},
    "YituTech/conv-bert-medium-small": {"do_lower_case": True},
    "YituTech/conv-bert-small": {"do_lower_case": True},
}
class 		SCREAMING_SNAKE_CASE_					(		_lowerCAmelCase					):




		"""simple docstring"""





		__lowercase				:    str		  =						VOCAB_FILES_NAMES
		__lowercase				:    Union[str, Any]		  =						PRETRAINED_VOCAB_FILES_MAP
		__lowercase				:    List[str]		  =						PRETRAINED_INIT_CONFIGURATION
		__lowercase				:    Tuple		  =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__lowercase				:    Union[str, Any]		  =						ConvBertTokenizer




		def __init__(	self				,    lowerCAmelCase__=None				,    lowerCAmelCase__=None				,    lowerCAmelCase__=True				,    lowerCAmelCase__="[UNK]"				,    lowerCAmelCase__="[SEP]"				,    lowerCAmelCase__="[PAD]"				,    lowerCAmelCase__="[CLS]"				,    lowerCAmelCase__="[MASK]"				,    lowerCAmelCase__=True				,    lowerCAmelCase__=None				,    **lowerCAmelCase__				,    ):
				super().__init__(
				    SCREAMING_SNAKE_CASE__				,    tokenizer_file=SCREAMING_SNAKE_CASE__				,    do_lower_case=SCREAMING_SNAKE_CASE__				,    unk_token=SCREAMING_SNAKE_CASE__				,    sep_token=SCREAMING_SNAKE_CASE__				,    pad_token=SCREAMING_SNAKE_CASE__				,    cls_token=SCREAMING_SNAKE_CASE__				,    mask_token=SCREAMING_SNAKE_CASE__				,    tokenize_chinese_chars=SCREAMING_SNAKE_CASE__				,    strip_accents=SCREAMING_SNAKE_CASE__				,    **SCREAMING_SNAKE_CASE__				,    )

				__SCREAMING_SNAKE_CASE				=					json.loads(self.backend_tokenizer.normalizer.__getstate__())
				if (
				    normalizer_state.get("""lowercase"""				,    SCREAMING_SNAKE_CASE__) != do_lower_case
				    or normalizer_state.get("""strip_accents"""				,    SCREAMING_SNAKE_CASE__) != strip_accents
				    or normalizer_state.get("""handle_chinese_chars"""				,    SCREAMING_SNAKE_CASE__) != tokenize_chinese_chars
				):
						__SCREAMING_SNAKE_CASE				=					getattr(SCREAMING_SNAKE_CASE__				,    normalizer_state.pop("""type"""))
						__SCREAMING_SNAKE_CASE				=					do_lower_case
						__SCREAMING_SNAKE_CASE				=					strip_accents
						__SCREAMING_SNAKE_CASE				=					tokenize_chinese_chars
						__SCREAMING_SNAKE_CASE				=					normalizer_class(**SCREAMING_SNAKE_CASE__)

				__SCREAMING_SNAKE_CASE				=					do_lower_case




		def       snake_case_	(	self				,    lowerCAmelCase__				,    lowerCAmelCase__=None):
				__SCREAMING_SNAKE_CASE				=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

				return output




		def       snake_case_	(	self				,    lowerCAmelCase__				,    lowerCAmelCase__ = None):
				__SCREAMING_SNAKE_CASE				=					[self.sep_token_id]
				__SCREAMING_SNAKE_CASE				=					[self.cls_token_id]
				if token_ids_a is None:
						return len(cls + token_ids_a + sep) * [0]
				return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]




		def       snake_case_	(	self				,    lowerCAmelCase__				,    lowerCAmelCase__ = None):
				__SCREAMING_SNAKE_CASE				=					self._tokenizer.model.save(SCREAMING_SNAKE_CASE__				,    name=SCREAMING_SNAKE_CASE__)
				return tuple(SCREAMING_SNAKE_CASE__)









"""simple docstring"""







from ..utils import (
    OptionalDependencyNotAvailable,
    is_flax_available,
    is_scipy_available,
    is_torch_available,
    is_torchsde_available,
)


try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			from ..utils.dummy_pt_objects import *  # noqa F403
else:
			from .scheduling_consistency_models import CMStochasticIterativeScheduler
			from .scheduling_ddim import DDIMScheduler
			from .scheduling_ddim_inverse import DDIMInverseScheduler
			from .scheduling_ddim_parallel import DDIMParallelScheduler
			from .scheduling_ddpm import DDPMScheduler
			from .scheduling_ddpm_parallel import DDPMParallelScheduler
			from .scheduling_deis_multistep import DEISMultistepScheduler
			from .scheduling_dpmsolver_multistep import DPMSolverMultistepScheduler
			from .scheduling_dpmsolver_multistep_inverse import DPMSolverMultistepInverseScheduler
			from .scheduling_dpmsolver_singlestep import DPMSolverSinglestepScheduler
			from .scheduling_euler_ancestral_discrete import EulerAncestralDiscreteScheduler
			from .scheduling_euler_discrete import EulerDiscreteScheduler
			from .scheduling_heun_discrete import HeunDiscreteScheduler
			from .scheduling_ipndm import IPNDMScheduler
			from .scheduling_k_dpm_2_ancestral_discrete import KDPMaAncestralDiscreteScheduler
			from .scheduling_k_dpm_2_discrete import KDPMaDiscreteScheduler
			from .scheduling_karras_ve import KarrasVeScheduler
			from .scheduling_pndm import PNDMScheduler
			from .scheduling_repaint import RePaintScheduler
			from .scheduling_sde_ve import ScoreSdeVeScheduler
			from .scheduling_sde_vp import ScoreSdeVpScheduler
			from .scheduling_unclip import UnCLIPScheduler
			from .scheduling_unipc_multistep import UniPCMultistepScheduler
			from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin
			from .scheduling_vq_diffusion import VQDiffusionScheduler

try:
			if not is_flax_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			from ..utils.dummy_flax_objects import *  # noqa F403
else:
			from .scheduling_ddim_flax import FlaxDDIMScheduler
			from .scheduling_ddpm_flax import FlaxDDPMScheduler
			from .scheduling_dpmsolver_multistep_flax import FlaxDPMSolverMultistepScheduler
			from .scheduling_karras_ve_flax import FlaxKarrasVeScheduler
			from .scheduling_lms_discrete_flax import FlaxLMSDiscreteScheduler
			from .scheduling_pndm_flax import FlaxPNDMScheduler
			from .scheduling_sde_ve_flax import FlaxScoreSdeVeScheduler
			from .scheduling_utils_flax import (
			    FlaxKarrasDiffusionSchedulers,
			    FlaxSchedulerMixin,
			    FlaxSchedulerOutput,
			    broadcast_to_shape_from_left,
			)


try:
			if not (is_torch_available() and is_scipy_available()):
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			from ..utils.dummy_torch_and_scipy_objects import *  # noqa F403
else:
			from .scheduling_lms_discrete import LMSDiscreteScheduler

try:
			if not (is_torch_available() and is_torchsde_available()):
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			from ..utils.dummy_torch_and_torchsde_objects import *  # noqa F403
else:
			from .scheduling_dpmsolver_sde import DPMSolverSDEScheduler




