



"""simple docstring"""

import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin
class     lowerCAmelCase__						(	_lowerCamelCase							,							unittest.TestCase ):


	'''simple docstring'''







	_SCREAMING_SNAKE_CASE	:		Union[str, Any]  						= ProphetNetTokenizer
	_SCREAMING_SNAKE_CASE	:		List[str]  						= False





	def 			_lowerCAmelCase     (      self				:       str    )						->	Tuple:





					"""simple docstring"""


					super().setUp()

					SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     [
					    '[UNK]',
					    '[CLS]',
					    '[SEP]',
					    '[PAD]',
					    '[MASK]',
					    'want',
					    '##want',
					    '##ed',
					    'wa',
					    'un',
					    'runn',
					    '##ing',
					    ',',
					    'low',
					    'lowest',
					]
					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     os.path.join(self.tmpdirname     ,       VOCAB_FILES_NAMES['vocab_file']    )
					with open(self.vocab_file     ,       'w'     ,       encoding='utf-8'    ) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]    )    )





	def 			_lowerCAmelCase     (      self				:       Tuple     ,       _SCREAMING_SNAKE_CASE				:       List[Any]    )						->	str:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Any							=     'UNwant\u00E9d,running'
					SCREAMING_SNAKE_CASE						:							Tuple							=     'unwanted, running'
					return input_text, output_text





	def 			_lowerCAmelCase     (      self				:       Optional[Any]    )						->	Union[str, Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     self.tokenizer_class(self.vocab_file    )

					SCREAMING_SNAKE_CASE						:							Any							=     tokenizer.tokenize('UNwant\u00E9d,running'    )
					self.assertListEqual(_SCREAMING_SNAKE_CASE     ,       ['un', '##want', '##ed', ',', 'runn', '##ing']    )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE    )     ,       [9, 6, 7, 12, 10, 11]    )





	def 			_lowerCAmelCase     (      self				:       int    )						->	int:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Optional[int]							=     BasicTokenizer()

					self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'    )     ,       ['ah', '\u535A', '\u63A8', 'zz']    )





	def 			_lowerCAmelCase     (      self				:       List[str]    )						->	Optional[int]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							int							=     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE    )

					self.assertListEqual(
					    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    )     ,       ['hello', '!', 'how', 'are', 'you', '?']    )
					self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )     ,       ['hello']    )





	def 			_lowerCAmelCase     (      self				:       Any    )						->	Optional[Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Optional[int]							=     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE     ,       strip_accents=_SCREAMING_SNAKE_CASE    )

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )     ,       ['hällo', '!', 'how', 'are', 'you', '?']    )
					self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )     ,       ['h\u00E9llo']    )





	def 			_lowerCAmelCase     (      self				:       Any    )						->	List[str]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE     ,       strip_accents=_SCREAMING_SNAKE_CASE    )

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )     ,       ['hallo', '!', 'how', 'are', 'you', '?']    )
					self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )     ,       ['hello']    )





	def 			_lowerCAmelCase     (      self				:       Union[str, Any]    )						->	List[Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							int							=     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE    )

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )     ,       ['hallo', '!', 'how', 'are', 'you', '?']    )
					self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )     ,       ['hello']    )





	def 			_lowerCAmelCase     (      self				:       Optional[int]    )						->	Optional[int]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Tuple							=     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE    )

					self.assertListEqual(
					    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    )     ,       ['HeLLo', '!', 'how', 'Are', 'yoU', '?']    )





	def 			_lowerCAmelCase     (      self				:       int    )						->	Union[str, Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							List[Any]							=     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE     ,       strip_accents=_SCREAMING_SNAKE_CASE    )

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )     ,       ['HäLLo', '!', 'how', 'Are', 'yoU', '?']    )





	def 			_lowerCAmelCase     (      self				:       List[str]    )						->	Tuple:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE     ,       strip_accents=_SCREAMING_SNAKE_CASE    )

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )     ,       ['HaLLo', '!', 'how', 'Are', 'yoU', '?']    )





	def 			_lowerCAmelCase     (      self				:       Any    )						->	Union[str, Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Optional[int]							=     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE     ,       never_split=['[UNK]']    )

					self.assertListEqual(
					    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'    )     ,       ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']    )





	def 			_lowerCAmelCase     (      self				:       List[Any]    )						->	str:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Tuple							=     ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

					SCREAMING_SNAKE_CASE						:							List[Any]							=     {}
					for i, token in enumerate(_SCREAMING_SNAKE_CASE    ):
									SCREAMING_SNAKE_CASE						:							List[Any]							=     i
					SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     WordpieceTokenizer(vocab=_SCREAMING_SNAKE_CASE     ,       unk_token='[UNK]'    )

					self.assertListEqual(tokenizer.tokenize(''    )     ,       []    )

					self.assertListEqual(tokenizer.tokenize('unwanted running'    )     ,       ['un', '##want', '##ed', 'runn', '##ing']    )

					self.assertListEqual(tokenizer.tokenize('unwantedX running'    )     ,       ['[UNK]', 'runn', '##ing']    )





	@require_torch
	def 			_lowerCAmelCase     (      self				:       Any    )						->	Optional[int]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Tuple							=     self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'    )

					SCREAMING_SNAKE_CASE						:							int							=     ['A long paragraph for summarization.', 'Another paragraph for summarization.']
					SCREAMING_SNAKE_CASE						:							str							=     [1_037, 2_146, 20_423, 2_005, 7_680, 7_849, 3_989, 1_012, 102]
					SCREAMING_SNAKE_CASE						:							Any							=     tokenizer(_SCREAMING_SNAKE_CASE     ,       padding=_SCREAMING_SNAKE_CASE     ,       return_tensors='pt'    )
					self.assertIsInstance(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE    )
					SCREAMING_SNAKE_CASE						:							Tuple							=     list(batch.input_ids.numpy()[0]    )
					self.assertListEqual(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE    )

					self.assertEqual((2, 9)     ,       batch.input_ids.shape    )
					self.assertEqual((2, 9)     ,       batch.attention_mask.shape    )





	def 			_lowerCAmelCase     (      self				:       Union[str, Any]    )						->	Tuple:





					"""simple docstring"""


					self.assertTrue(_is_whitespace(' '    )    )
					self.assertTrue(_is_whitespace('\t'    )    )
					self.assertTrue(_is_whitespace('\r'    )    )
					self.assertTrue(_is_whitespace('\n'    )    )
					self.assertTrue(_is_whitespace('\u00A0'    )    )

					self.assertFalse(_is_whitespace('A'    )    )
					self.assertFalse(_is_whitespace('-'    )    )





	def 			_lowerCAmelCase     (      self				:       int    )						->	Optional[int]:





					"""simple docstring"""


					self.assertTrue(_is_control('\u0005'    )    )

					self.assertFalse(_is_control('A'    )    )
					self.assertFalse(_is_control(' '    )    )
					self.assertFalse(_is_control('\t'    )    )
					self.assertFalse(_is_control('\r'    )    )





	def 			_lowerCAmelCase     (      self				:       Tuple    )						->	List[Any]:





					"""simple docstring"""


					self.assertTrue(_is_punctuation('-'    )    )
					self.assertTrue(_is_punctuation('$'    )    )
					self.assertTrue(_is_punctuation('`'    )    )
					self.assertTrue(_is_punctuation('.'    )    )

					self.assertFalse(_is_punctuation('A'    )    )
					self.assertFalse(_is_punctuation(' '    )    )







	@slow
	def 			_lowerCAmelCase     (      self				:       Optional[int]    )						->	Dict:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Any							=     self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'    )

					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     tokenizer.encode('sequence builders'     ,       add_special_tokens=_SCREAMING_SNAKE_CASE    )
					SCREAMING_SNAKE_CASE						:							List[Any]							=     tokenizer.encode('multi-sequence build'     ,       add_special_tokens=_SCREAMING_SNAKE_CASE    )

					SCREAMING_SNAKE_CASE						:							Any							=     tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE    )
					SCREAMING_SNAKE_CASE						:							List[str]							=     tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE    )

					assert encoded_sentence == text + [102]
					assert encoded_pair == text + [102] + text_a + [102]










"""simple docstring"""

A_						:    Optional[Any]				=      '0.18.2'

from .configuration_utils import ConfigMixin
from .utils import (
    OptionalDependencyNotAvailable,
    is_flax_available,
    is_inflect_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_k_diffusion_version,
    is_librosa_available,
    is_note_seq_available,
    is_onnx_available,
    is_scipy_available,
    is_torch_available,
    is_torchsde_available,
    is_transformers_available,
    is_transformers_version,
    is_unidecode_available,
    logging,
)


try:
							if not is_onnx_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							from .utils.dummy_onnx_objects import *  # noqa F403
else:
							from .pipelines import OnnxRuntimeModel

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							from .utils.dummy_pt_objects import *  # noqa F403
else:
							from .models import (
							    AutoencoderKL,
							    ControlNetModel,
							    ModelMixin,
							    PriorTransformer,
							    TaFilmDecoder,
							    TransformeraDModel,
							    UNetaDModel,
							    UNetaDConditionModel,
							    UNetaDModel,
							    UNetaDConditionModel,
							    VQModel,
							)
							from .optimization import (
							    get_constant_schedule,
							    get_constant_schedule_with_warmup,
							    get_cosine_schedule_with_warmup,
							    get_cosine_with_hard_restarts_schedule_with_warmup,
							    get_linear_schedule_with_warmup,
							    get_polynomial_decay_schedule_with_warmup,
							    get_scheduler,
							)
							from .pipelines import (
							    AudioPipelineOutput,
							    ConsistencyModelPipeline,
							    DanceDiffusionPipeline,
							    DDIMPipeline,
							    DDPMPipeline,
							    DiffusionPipeline,
							    DiTPipeline,
							    ImagePipelineOutput,
							    KarrasVePipeline,
							    LDMPipeline,
							    LDMSuperResolutionPipeline,
							    PNDMPipeline,
							    RePaintPipeline,
							    ScoreSdeVePipeline,
							)
							from .schedulers import (
							    CMStochasticIterativeScheduler,
							    DDIMInverseScheduler,
							    DDIMParallelScheduler,
							    DDIMScheduler,
							    DDPMParallelScheduler,
							    DDPMScheduler,
							    DEISMultistepScheduler,
							    DPMSolverMultistepInverseScheduler,
							    DPMSolverMultistepScheduler,
							    DPMSolverSinglestepScheduler,
							    EulerAncestralDiscreteScheduler,
							    EulerDiscreteScheduler,
							    HeunDiscreteScheduler,
							    IPNDMScheduler,
							    KarrasVeScheduler,
							    KDPMaAncestralDiscreteScheduler,
							    KDPMaDiscreteScheduler,
							    PNDMScheduler,
							    RePaintScheduler,
							    SchedulerMixin,
							    ScoreSdeVeScheduler,
							    UnCLIPScheduler,
							    UniPCMultistepScheduler,
							    VQDiffusionScheduler,
							)
							from .training_utils import EMAModel

try:
							if not (is_torch_available() and is_scipy_available()):
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							from .utils.dummy_torch_and_scipy_objects import *  # noqa F403
else:
							from .schedulers import LMSDiscreteScheduler

try:
							if not (is_torch_available() and is_torchsde_available()):
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							from .utils.dummy_torch_and_torchsde_objects import *  # noqa F403
else:
							from .schedulers import DPMSolverSDEScheduler

try:
							if not (is_torch_available() and is_transformers_available()):
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							from .utils.dummy_torch_and_transformers_objects import *  # noqa F403
else:
							from .pipelines import (
							    AltDiffusionImgaImgPipeline,
							    AltDiffusionPipeline,
							    AudioLDMPipeline,
							    CycleDiffusionPipeline,
							    IFImgaImgPipeline,
							    IFImgaImgSuperResolutionPipeline,
							    IFInpaintingPipeline,
							    IFInpaintingSuperResolutionPipeline,
							    IFPipeline,
							    IFSuperResolutionPipeline,
							    ImageTextPipelineOutput,
							    KandinskyImgaImgPipeline,
							    KandinskyInpaintPipeline,
							    KandinskyPipeline,
							    KandinskyPriorPipeline,
							    KandinskyVaaControlnetImgaImgPipeline,
							    KandinskyVaaControlnetPipeline,
							    KandinskyVaaImgaImgPipeline,
							    KandinskyVaaInpaintPipeline,
							    KandinskyVaaPipeline,
							    KandinskyVaaPriorEmbaEmbPipeline,
							    KandinskyVaaPriorPipeline,
							    LDMTextToImagePipeline,
							    PaintByExamplePipeline,
							    SemanticStableDiffusionPipeline,
							    ShapEImgaImgPipeline,
							    ShapEPipeline,
							    StableDiffusionAttendAndExcitePipeline,
							    StableDiffusionControlNetImgaImgPipeline,
							    StableDiffusionControlNetInpaintPipeline,
							    StableDiffusionControlNetPipeline,
							    StableDiffusionDepthaImgPipeline,
							    StableDiffusionDiffEditPipeline,
							    StableDiffusionImageVariationPipeline,
							    StableDiffusionImgaImgPipeline,
							    StableDiffusionInpaintPipeline,
							    StableDiffusionInpaintPipelineLegacy,
							    StableDiffusionInstructPixaPixPipeline,
							    StableDiffusionLatentUpscalePipeline,
							    StableDiffusionLDMaDPipeline,
							    StableDiffusionModelEditingPipeline,
							    StableDiffusionPanoramaPipeline,
							    StableDiffusionParadigmsPipeline,
							    StableDiffusionPipeline,
							    StableDiffusionPipelineSafe,
							    StableDiffusionPixaPixZeroPipeline,
							    StableDiffusionSAGPipeline,
							    StableDiffusionUpscalePipeline,
							    StableUnCLIPImgaImgPipeline,
							    StableUnCLIPPipeline,
							    TextToVideoSDPipeline,
							    TextToVideoZeroPipeline,
							    UnCLIPImageVariationPipeline,
							    UnCLIPPipeline,
							    UniDiffuserModel,
							    UniDiffuserPipeline,
							    UniDiffuserTextDecoder,
							    VersatileDiffusionDualGuidedPipeline,
							    VersatileDiffusionImageVariationPipeline,
							    VersatileDiffusionPipeline,
							    VersatileDiffusionTextToImagePipeline,
							    VideoToVideoSDPipeline,
							    VQDiffusionPipeline,
							)

try:
							if not (is_torch_available() and is_transformers_available() and is_invisible_watermark_available()):
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							from .utils.dummy_torch_and_transformers_and_invisible_watermark_objects import *  # noqa F403
else:
							from .pipelines import StableDiffusionXLImgaImgPipeline, StableDiffusionXLPipeline

try:
							if not (is_torch_available() and is_transformers_available() and is_k_diffusion_available()):
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							from .utils.dummy_torch_and_transformers_and_k_diffusion_objects import *  # noqa F403
else:
							from .pipelines import StableDiffusionKDiffusionPipeline

try:
							if not (is_torch_available() and is_transformers_available() and is_onnx_available()):
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							from .utils.dummy_torch_and_transformers_and_onnx_objects import *  # noqa F403
else:
							from .pipelines import (
							    OnnxStableDiffusionImgaImgPipeline,
							    OnnxStableDiffusionInpaintPipeline,
							    OnnxStableDiffusionInpaintPipelineLegacy,
							    OnnxStableDiffusionPipeline,
							    OnnxStableDiffusionUpscalePipeline,
							    StableDiffusionOnnxPipeline,
							)

try:
							if not (is_torch_available() and is_librosa_available()):
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							from .utils.dummy_torch_and_librosa_objects import *  # noqa F403
else:
							from .pipelines import AudioDiffusionPipeline, Mel

try:
							if not (is_transformers_available() and is_torch_available() and is_note_seq_available()):
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							from .utils.dummy_transformers_and_torch_and_note_seq_objects import *  # noqa F403
else:
							from .pipelines import SpectrogramDiffusionPipeline

try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							from .utils.dummy_flax_objects import *  # noqa F403
else:
							from .models.controlnet_flax import FlaxControlNetModel
							from .models.modeling_flax_utils import FlaxModelMixin
							from .models.unet_ad_condition_flax import FlaxUNetaDConditionModel
							from .models.vae_flax import FlaxAutoencoderKL
							from .pipelines import FlaxDiffusionPipeline
							from .schedulers import (
							    FlaxDDIMScheduler,
							    FlaxDDPMScheduler,
							    FlaxDPMSolverMultistepScheduler,
							    FlaxKarrasVeScheduler,
							    FlaxLMSDiscreteScheduler,
							    FlaxPNDMScheduler,
							    FlaxSchedulerMixin,
							    FlaxScoreSdeVeScheduler,
							)


try:
							if not (is_flax_available() and is_transformers_available()):
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							from .utils.dummy_flax_and_transformers_objects import *  # noqa F403
else:
							from .pipelines import (
							    FlaxStableDiffusionControlNetPipeline,
							    FlaxStableDiffusionImgaImgPipeline,
							    FlaxStableDiffusionInpaintPipeline,
							    FlaxStableDiffusionPipeline,
							)

try:
							if not (is_note_seq_available()):
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							from .utils.dummy_note_seq_objects import *  # noqa F403
else:
							from .pipelines import MidiProcessor





