






'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


__UpperCamelCase      :       Tuple      			=      logging.get_logger(__name__)


__UpperCamelCase      :       Any      			=      {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
__UpperCamelCase      :       Optional[Any]      			=      {
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json""",
    },
}

__UpperCamelCase      :       Optional[int]      			=      {
    """RUCAIBox/mvp""": 1024,
}
class 							__SCREAMING_SNAKE_CASE       (							_lowerCAmelCase     ):
      __a  					=VOCAB_FILES_NAMES
      __a  					=PRETRAINED_VOCAB_FILES_MAP
      __a  					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __a  					=["input_ids", "attention_mask"]
      __a  					=MvpTokenizer




      def __init__(  self ,   lowerCamelCase=None ,   lowerCamelCase=None ,   lowerCamelCase=None ,   lowerCamelCase="replace" ,   lowerCamelCase="<s>" ,   lowerCamelCase="</s>" ,   lowerCamelCase="</s>" ,   lowerCamelCase="<s>" ,   lowerCamelCase="<unk>" ,   lowerCamelCase="<pad>" ,   lowerCamelCase="<mask>" ,   lowerCamelCase=False ,   lowerCamelCase=True ,   **lowerCamelCase ,   )							->str:



         '''simple docstring'''
         super().__init__(
             lowerCamelCase ,   lowerCamelCase ,   tokenizer_file=lowerCamelCase ,   errors=lowerCamelCase ,   bos_token=lowerCamelCase ,   eos_token=lowerCamelCase ,   sep_token=lowerCamelCase ,   cls_token=lowerCamelCase ,   unk_token=lowerCamelCase ,   pad_token=lowerCamelCase ,   mask_token=lowerCamelCase ,   add_prefix_space=lowerCamelCase ,   trim_offsets=lowerCamelCase ,   **lowerCamelCase ,   )

         __a				     =    json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
         if pre_tok_state.get('add_prefix_space' ,   lowerCamelCase      ) != add_prefix_space:
            __a				     =    getattr(lowerCamelCase ,   pre_tok_state.pop('type'      )      )
            __a				     =    add_prefix_space
            __a				     =    pre_tok_class(**lowerCamelCase      )

         __a				     =    add_prefix_space

         # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
         __a				     =    'post_processor'
         __a				     =    getattr(self.backend_tokenizer ,   lowerCamelCase ,   lowerCamelCase      )
         if tokenizer_component_instance:
            __a				     =    json.loads(tokenizer_component_instance.__getstate__()      )

            # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
            if "sep" in state:
               __a				     =    tuple(state['sep']      )
            if "cls" in state:
               __a				     =    tuple(state['cls']      )

            __a				     =    False

            if state.get('add_prefix_space' ,   lowerCamelCase      ) != add_prefix_space:
               __a				     =    add_prefix_space
               __a				     =    True

            if state.get('trim_offsets' ,   lowerCamelCase      ) != trim_offsets:
               __a				     =    trim_offsets
               __a				     =    True

            if changes_to_apply:
               __a				     =    getattr(lowerCamelCase ,   state.pop('type'      )      )
               __a				     =    component_class(**lowerCamelCase      )
               setattr(self.backend_tokenizer ,   lowerCamelCase ,   lowerCamelCase      )




      @property
      def  __UpperCamelCase							(  self      )							->str:



         '''simple docstring'''
         if self._mask_token is None:
            if self.verbose:
               logger.error('Using mask_token, but it is not set yet.'      )
            return None
         return str(self._mask_token      )




      @mask_token.setter
      def  __UpperCamelCase							(  self ,   lowerCamelCase      )							->List[str]:



         '''simple docstring'''
         __a				     =    AddedToken(lowerCamelCase ,   lstrip=lowerCamelCase ,   rstrip=lowerCamelCase      ) if isinstance(lowerCamelCase ,   lowerCamelCase      ) else value
         __a				     =    value




      def  __UpperCamelCase							(  self ,   *lowerCamelCase ,   **lowerCamelCase      )							->BatchEncoding:



         '''simple docstring'''
         __a				     =    kwargs.get('is_split_into_words' ,   lowerCamelCase      )

         if is_split_into_words and not self.add_prefix_space:
            raise ValueError(
                F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
                'to use it with pretokenized inputs.'      )

         return super()._batch_encode_plus(*lowerCamelCase ,   **lowerCamelCase      )




      def  __UpperCamelCase							(  self ,   *lowerCamelCase ,   **lowerCamelCase      )							->BatchEncoding:



         '''simple docstring'''
         __a				     =    kwargs.get('is_split_into_words' ,   lowerCamelCase      )

         if is_split_into_words and not self.add_prefix_space:
            raise ValueError(
                F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
                'to use it with pretokenized inputs.'      )

         return super()._encode_plus(*lowerCamelCase ,   **lowerCamelCase      )




      def  __UpperCamelCase							(  self ,   lowerCamelCase ,   lowerCamelCase = None      )							->Tuple[str]:



         '''simple docstring'''
         __a				     =    self._tokenizer.model.save(lowerCamelCase ,   name=lowerCamelCase      )
         return tuple(lowerCamelCase      )




      def  __UpperCamelCase							(  self ,   lowerCamelCase ,   lowerCamelCase=None      )							->Dict:



         '''simple docstring'''
         __a				     =    [self.bos_token_id] + token_ids_a + [self.eos_token_id]
         if token_ids_a is None:
            return output

         return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






      def  __UpperCamelCase							(  self ,   lowerCamelCase ,   lowerCamelCase = None      )							->List[int]:



         '''simple docstring'''
         __a				     =    [self.sep_token_id]
         __a				     =    [self.cls_token_id]

         if token_ids_a is None:
            return len(cls + token_ids_a + sep      ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]







'''simple docstring'''



__UpperCamelCase      :       Tuple      			=      """0.18.2"""

from .configuration_utils import ConfigMixin
from .utils import (
    OptionalDependencyNotAvailable,
    is_flax_available,
    is_inflect_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_k_diffusion_version,
    is_librosa_available,
    is_note_seq_available,
    is_onnx_available,
    is_scipy_available,
    is_torch_available,
    is_torchsde_available,
    is_transformers_available,
    is_transformers_version,
    is_unidecode_available,
    logging,
)


try:
 if not is_onnx_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from .utils.dummy_onnx_objects import *  # noqa F403
else:
 from .pipelines import OnnxRuntimeModel

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from .utils.dummy_pt_objects import *  # noqa F403
else:
 from .models import (
     AutoencoderKL,
     ControlNetModel,
     ModelMixin,
     PriorTransformer,
     TaFilmDecoder,
     TransformeraDModel,
     UNetaDModel,
     UNetaDConditionModel,
     UNetaDModel,
     UNetaDConditionModel,
     VQModel,
 )
 from .optimization import (
     get_constant_schedule,
     get_constant_schedule_with_warmup,
     get_cosine_schedule_with_warmup,
     get_cosine_with_hard_restarts_schedule_with_warmup,
     get_linear_schedule_with_warmup,
     get_polynomial_decay_schedule_with_warmup,
     get_scheduler,
 )
 from .pipelines import (
     AudioPipelineOutput,
     ConsistencyModelPipeline,
     DanceDiffusionPipeline,
     DDIMPipeline,
     DDPMPipeline,
     DiffusionPipeline,
     DiTPipeline,
     ImagePipelineOutput,
     KarrasVePipeline,
     LDMPipeline,
     LDMSuperResolutionPipeline,
     PNDMPipeline,
     RePaintPipeline,
     ScoreSdeVePipeline,
 )
 from .schedulers import (
     CMStochasticIterativeScheduler,
     DDIMInverseScheduler,
     DDIMParallelScheduler,
     DDIMScheduler,
     DDPMParallelScheduler,
     DDPMScheduler,
     DEISMultistepScheduler,
     DPMSolverMultistepInverseScheduler,
     DPMSolverMultistepScheduler,
     DPMSolverSinglestepScheduler,
     EulerAncestralDiscreteScheduler,
     EulerDiscreteScheduler,
     HeunDiscreteScheduler,
     IPNDMScheduler,
     KarrasVeScheduler,
     KDPMaAncestralDiscreteScheduler,
     KDPMaDiscreteScheduler,
     PNDMScheduler,
     RePaintScheduler,
     SchedulerMixin,
     ScoreSdeVeScheduler,
     UnCLIPScheduler,
     UniPCMultistepScheduler,
     VQDiffusionScheduler,
 )
 from .training_utils import EMAModel

try:
 if not (is_torch_available() and is_scipy_available()):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from .utils.dummy_torch_and_scipy_objects import *  # noqa F403
else:
 from .schedulers import LMSDiscreteScheduler

try:
 if not (is_torch_available() and is_torchsde_available()):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from .utils.dummy_torch_and_torchsde_objects import *  # noqa F403
else:
 from .schedulers import DPMSolverSDEScheduler

try:
 if not (is_torch_available() and is_transformers_available()):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from .utils.dummy_torch_and_transformers_objects import *  # noqa F403
else:
 from .pipelines import (
     AltDiffusionImgaImgPipeline,
     AltDiffusionPipeline,
     AudioLDMPipeline,
     CycleDiffusionPipeline,
     IFImgaImgPipeline,
     IFImgaImgSuperResolutionPipeline,
     IFInpaintingPipeline,
     IFInpaintingSuperResolutionPipeline,
     IFPipeline,
     IFSuperResolutionPipeline,
     ImageTextPipelineOutput,
     KandinskyImgaImgPipeline,
     KandinskyInpaintPipeline,
     KandinskyPipeline,
     KandinskyPriorPipeline,
     KandinskyVaaControlnetImgaImgPipeline,
     KandinskyVaaControlnetPipeline,
     KandinskyVaaImgaImgPipeline,
     KandinskyVaaInpaintPipeline,
     KandinskyVaaPipeline,
     KandinskyVaaPriorEmbaEmbPipeline,
     KandinskyVaaPriorPipeline,
     LDMTextToImagePipeline,
     PaintByExamplePipeline,
     SemanticStableDiffusionPipeline,
     ShapEImgaImgPipeline,
     ShapEPipeline,
     StableDiffusionAttendAndExcitePipeline,
     StableDiffusionControlNetImgaImgPipeline,
     StableDiffusionControlNetInpaintPipeline,
     StableDiffusionControlNetPipeline,
     StableDiffusionDepthaImgPipeline,
     StableDiffusionDiffEditPipeline,
     StableDiffusionImageVariationPipeline,
     StableDiffusionImgaImgPipeline,
     StableDiffusionInpaintPipeline,
     StableDiffusionInpaintPipelineLegacy,
     StableDiffusionInstructPixaPixPipeline,
     StableDiffusionLatentUpscalePipeline,
     StableDiffusionLDMaDPipeline,
     StableDiffusionModelEditingPipeline,
     StableDiffusionPanoramaPipeline,
     StableDiffusionParadigmsPipeline,
     StableDiffusionPipeline,
     StableDiffusionPipelineSafe,
     StableDiffusionPixaPixZeroPipeline,
     StableDiffusionSAGPipeline,
     StableDiffusionUpscalePipeline,
     StableUnCLIPImgaImgPipeline,
     StableUnCLIPPipeline,
     TextToVideoSDPipeline,
     TextToVideoZeroPipeline,
     UnCLIPImageVariationPipeline,
     UnCLIPPipeline,
     UniDiffuserModel,
     UniDiffuserPipeline,
     UniDiffuserTextDecoder,
     VersatileDiffusionDualGuidedPipeline,
     VersatileDiffusionImageVariationPipeline,
     VersatileDiffusionPipeline,
     VersatileDiffusionTextToImagePipeline,
     VideoToVideoSDPipeline,
     VQDiffusionPipeline,
 )

try:
 if not (is_torch_available() and is_transformers_available() and is_invisible_watermark_available()):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from .utils.dummy_torch_and_transformers_and_invisible_watermark_objects import *  # noqa F403
else:
 from .pipelines import StableDiffusionXLImgaImgPipeline, StableDiffusionXLPipeline

try:
 if not (is_torch_available() and is_transformers_available() and is_k_diffusion_available()):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from .utils.dummy_torch_and_transformers_and_k_diffusion_objects import *  # noqa F403
else:
 from .pipelines import StableDiffusionKDiffusionPipeline

try:
 if not (is_torch_available() and is_transformers_available() and is_onnx_available()):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from .utils.dummy_torch_and_transformers_and_onnx_objects import *  # noqa F403
else:
 from .pipelines import (
     OnnxStableDiffusionImgaImgPipeline,
     OnnxStableDiffusionInpaintPipeline,
     OnnxStableDiffusionInpaintPipelineLegacy,
     OnnxStableDiffusionPipeline,
     OnnxStableDiffusionUpscalePipeline,
     StableDiffusionOnnxPipeline,
 )

try:
 if not (is_torch_available() and is_librosa_available()):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from .utils.dummy_torch_and_librosa_objects import *  # noqa F403
else:
 from .pipelines import AudioDiffusionPipeline, Mel

try:
 if not (is_transformers_available() and is_torch_available() and is_note_seq_available()):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from .utils.dummy_transformers_and_torch_and_note_seq_objects import *  # noqa F403
else:
 from .pipelines import SpectrogramDiffusionPipeline

try:
 if not is_flax_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from .utils.dummy_flax_objects import *  # noqa F403
else:
 from .models.controlnet_flax import FlaxControlNetModel
 from .models.modeling_flax_utils import FlaxModelMixin
 from .models.unet_ad_condition_flax import FlaxUNetaDConditionModel
 from .models.vae_flax import FlaxAutoencoderKL
 from .pipelines import FlaxDiffusionPipeline
 from .schedulers import (
     FlaxDDIMScheduler,
     FlaxDDPMScheduler,
     FlaxDPMSolverMultistepScheduler,
     FlaxKarrasVeScheduler,
     FlaxLMSDiscreteScheduler,
     FlaxPNDMScheduler,
     FlaxSchedulerMixin,
     FlaxScoreSdeVeScheduler,
 )


try:
 if not (is_flax_available() and is_transformers_available()):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from .utils.dummy_flax_and_transformers_objects import *  # noqa F403
else:
 from .pipelines import (
     FlaxStableDiffusionControlNetPipeline,
     FlaxStableDiffusionImgaImgPipeline,
     FlaxStableDiffusionInpaintPipeline,
     FlaxStableDiffusionPipeline,
 )

try:
 if not (is_note_seq_available()):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from .utils.dummy_note_seq_objects import *  # noqa F403
else:
 from .pipelines import MidiProcessor