


import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler, UNetaDConditionModel
from diffusers.pipelines.stable_diffusion_safe import StableDiffusionPipelineSafe as StableDiffusionPipeline
from diffusers.utils import floats_tensor, nightly, torch_device
from diffusers.utils.testing_utils import require_torch_gpu

class     A__   (       unittest.TestCase      ):







    """simple docstring"""

    def 						a_     (			self       ):
         # clean up the VRAM after each test
         super().tearDown()
         gc.collect()
         torch.cuda.empty_cache()
    @property
    def 						a_     (			self       ):
         snake_case											=							1
         snake_case											=							3
         snake_case											=							(3_2, 3_2)

         snake_case											=							floats_tensor((batch_size, num_channels) + sizes     ,   rng=random.Random(0       )       ).to(__snake_case       )
         return image
    @property
    def 						a_     (			self       ):
         torch.manual_seed(0       )
         snake_case											=							UNetaDConditionModel(
             block_out_channels=(3_2, 6_4)     ,   layers_per_block=2     ,   sample_size=3_2     ,   in_channels=4     ,   out_channels=4     ,   down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''')     ,   up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''')     ,   cross_attention_dim=3_2     ,   )
         return model
    @property
    def 						a_     (			self       ):
         torch.manual_seed(0       )
         snake_case											=							AutoencoderKL(
             block_out_channels=[3_2, 6_4]     ,   in_channels=3     ,   out_channels=3     ,   down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']     ,   up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']     ,   latent_channels=4     ,   )
         return model
    @property
    def 						a_     (			self       ):
         torch.manual_seed(0       )
         snake_case											=							CLIPTextConfig(
             bos_token_id=0     ,   eos_token_id=2     ,   hidden_size=3_2     ,   intermediate_size=3_7     ,   layer_norm_eps=1E-05     ,   num_attention_heads=4     ,   num_hidden_layers=5     ,   pad_token_id=1     ,   vocab_size=1_0_0_0     ,   )
         return CLIPTextModel(__snake_case       )
    @property
    def 						a_     (			self       ):
         def extract(*__snake_case     ,   **__snake_case       ):

              class     A__   :







                  """simple docstring"""

                  def __init__(			self       ):
                       snake_case											=							torch.ones([0]       )




                  def 						a_     (			self     ,   __snake_case       ):
                       self.pixel_values.to(__snake_case       )
                       return self






              return Out()

         return extract
    def 						a_     (			self       ):
         snake_case											=							'''cpu'''  # ensure determinism for the device-dependent torch.Generator
         snake_case											=							self.dummy_cond_unet
         snake_case											=							DDIMScheduler(
             beta_start=0.0_0085     ,   beta_end=0.012     ,   beta_schedule='''scaled_linear'''     ,   clip_sample=__snake_case     ,   set_alpha_to_one=__snake_case     ,   )

         snake_case											=							self.dummy_vae
         snake_case											=							self.dummy_text_encoder
         snake_case											=							CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''       )

         # make sure here that pndm scheduler skips prk
         snake_case											=							StableDiffusionPipeline(
             unet=__snake_case     ,   scheduler=__snake_case     ,   vae=__snake_case     ,   text_encoder=__snake_case     ,   tokenizer=__snake_case     ,   safety_checker=__snake_case     ,   feature_extractor=self.dummy_extractor     ,   )
         snake_case											=							sd_pipe.to(__snake_case       )
         sd_pipe.set_progress_bar_config(disable=__snake_case       )

         snake_case											=							'''A painting of a squirrel eating a burger'''

         snake_case											=							torch.Generator(device=__snake_case       ).manual_seed(0       )
         snake_case											=							sd_pipe([prompt]     ,   generator=__snake_case     ,   guidance_scale=6.0     ,   num_inference_steps=2     ,   output_type='''np'''       )
         snake_case											=							output.images

         snake_case											=							torch.Generator(device=__snake_case       ).manual_seed(0       )
         snake_case											=							sd_pipe(
             [prompt]     ,   generator=__snake_case     ,   guidance_scale=6.0     ,   num_inference_steps=2     ,   output_type='''np'''     ,   return_dict=__snake_case     ,   )[0]

         snake_case											=							image[0, -3:, -3:, -1]
         snake_case											=							image_from_tuple[0, -3:, -3:, -1]

         assert image.shape == (1, 6_4, 6_4, 3)
         snake_case											=							np.array([0.5756, 0.6118, 0.5005, 0.5041, 0.5471, 0.4726, 0.4976, 0.4865, 0.4864]       )

         assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-2
         assert np.abs(image_from_tuple_slice.flatten() - expected_slice       ).max() < 1E-2
    def 						a_     (			self       ):
         snake_case											=							'''cpu'''  # ensure determinism for the device-dependent torch.Generator
         snake_case											=							self.dummy_cond_unet
         snake_case											=							PNDMScheduler(skip_prk_steps=__snake_case       )
         snake_case											=							self.dummy_vae
         snake_case											=							self.dummy_text_encoder
         snake_case											=							CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''       )

         # make sure here that pndm scheduler skips prk
         snake_case											=							StableDiffusionPipeline(
             unet=__snake_case     ,   scheduler=__snake_case     ,   vae=__snake_case     ,   text_encoder=__snake_case     ,   tokenizer=__snake_case     ,   safety_checker=__snake_case     ,   feature_extractor=self.dummy_extractor     ,   )
         snake_case											=							sd_pipe.to(__snake_case       )
         sd_pipe.set_progress_bar_config(disable=__snake_case       )

         snake_case											=							'''A painting of a squirrel eating a burger'''
         snake_case											=							torch.Generator(device=__snake_case       ).manual_seed(0       )
         snake_case											=							sd_pipe([prompt]     ,   generator=__snake_case     ,   guidance_scale=6.0     ,   num_inference_steps=2     ,   output_type='''np'''       )

         snake_case											=							output.images

         snake_case											=							torch.Generator(device=__snake_case       ).manual_seed(0       )
         snake_case											=							sd_pipe(
             [prompt]     ,   generator=__snake_case     ,   guidance_scale=6.0     ,   num_inference_steps=2     ,   output_type='''np'''     ,   return_dict=__snake_case     ,   )[0]

         snake_case											=							image[0, -3:, -3:, -1]
         snake_case											=							image_from_tuple[0, -3:, -3:, -1]

         assert image.shape == (1, 6_4, 6_4, 3)
         snake_case											=							np.array([0.5125, 0.5716, 0.4828, 0.5060, 0.5650, 0.4768, 0.5185, 0.4895, 0.4993]       )

         assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-2
         assert np.abs(image_from_tuple_slice.flatten() - expected_slice       ).max() < 1E-2
    def 						a_     (			self       ):
         snake_case											=							StableDiffusionPipeline.from_pretrained(
             '''hf-internal-testing/tiny-stable-diffusion-lms-pipe'''     ,   safety_checker=__snake_case       )
         assert isinstance(__snake_case     ,   __snake_case       )
         assert isinstance(pipe.scheduler     ,   __snake_case       )
         assert pipe.safety_checker is None

         snake_case											=							pipe('''example prompt'''     ,   num_inference_steps=2       ).images[0]
         assert image is not None

         # check that there's no error when saving a pipeline with one of the models being None
         with tempfile.TemporaryDirectory() as tmpdirname:
              pipe.save_pretrained(__snake_case       )
              snake_case											=							StableDiffusionPipeline.from_pretrained(__snake_case       )

         # sanity check that the pipeline still works
         assert pipe.safety_checker is None
         snake_case											=							pipe('''example prompt'''     ,   num_inference_steps=2       ).images[0]
         assert image is not None




    @unittest.skipIf(torch_device != '''cuda'''     ,   '''This test requires a GPU'''       )
    def 						a_     (			self       ):
         snake_case											=							self.dummy_cond_unet
         snake_case											=							PNDMScheduler(skip_prk_steps=__snake_case       )
         snake_case											=							self.dummy_vae
         snake_case											=							self.dummy_text_encoder
         snake_case											=							CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''       )

         # put models in fp16
         snake_case											=							unet.half()
         snake_case											=							vae.half()
         snake_case											=							bert.half()

         # make sure here that pndm scheduler skips prk
         snake_case											=							StableDiffusionPipeline(
             unet=__snake_case     ,   scheduler=__snake_case     ,   vae=__snake_case     ,   text_encoder=__snake_case     ,   tokenizer=__snake_case     ,   safety_checker=__snake_case     ,   feature_extractor=self.dummy_extractor     ,   )
         snake_case											=							sd_pipe.to(__snake_case       )
         sd_pipe.set_progress_bar_config(disable=__snake_case       )

         snake_case											=							'''A painting of a squirrel eating a burger'''
         snake_case											=							sd_pipe([prompt]     ,   num_inference_steps=2     ,   output_type='''np'''       ).images

         assert image.shape == (1, 6_4, 6_4, 3)






@nightly
@require_torch_gpu
class     A__   (       unittest.TestCase      ):







    """simple docstring"""

    def 						a_     (			self       ):
         # clean up the VRAM after each test
         super().tearDown()
         gc.collect()
         torch.cuda.empty_cache()
    def 						a_     (			self       ):
         snake_case											=							StableDiffusionPipeline.from_pretrained('''runwayml/stable-diffusion-v1-5'''     ,   safety_checker=__snake_case       )
         snake_case											=							LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config       )
         snake_case											=							sd_pipe.to(__snake_case       )
         sd_pipe.set_progress_bar_config(disable=__snake_case       )

         snake_case											=							(
             '''portrait of girl with smokey eyes makeup in abandoned hotel, grange clothes, redshift, wide high angle'''
             ''' coloured polaroid photograph with flash, kodak film, hyper real, stunning moody cinematography, with'''
             ''' anamorphic lenses, by maripol, fallen angels by wong kar - wai, style of suspiria and neon demon and'''
             ''' children from bahnhof zoo, detailed '''
         )
         snake_case											=							4_0_0_3_6_6_0_3_4_6
         snake_case											=							7

         # without safety guidance (sld_guidance_scale = 0)
         snake_case											=							torch.manual_seed(__snake_case       )
         snake_case											=							sd_pipe(
             [prompt]     ,   generator=__snake_case     ,   guidance_scale=__snake_case     ,   num_inference_steps=5_0     ,   output_type='''np'''     ,   width=5_1_2     ,   height=5_1_2     ,   sld_guidance_scale=0     ,   )

         snake_case											=							output.images
         snake_case											=							image[0, -3:, -3:, -1]
         snake_case											=							[0.2278, 0.2231, 0.2249, 0.2333, 0.2303, 0.1885, 0.2273, 0.2144, 0.2176]

         assert image.shape == (1, 5_1_2, 5_1_2, 3)

         assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-2

         # without safety guidance (strong configuration)
         snake_case											=							torch.manual_seed(__snake_case       )
         snake_case											=							sd_pipe(
             [prompt]     ,   generator=__snake_case     ,   guidance_scale=__snake_case     ,   num_inference_steps=5_0     ,   output_type='''np'''     ,   width=5_1_2     ,   height=5_1_2     ,   sld_guidance_scale=2_0_0_0     ,   sld_warmup_steps=7     ,   sld_threshold=0.025     ,   sld_momentum_scale=0.5     ,   sld_mom_beta=0.7     ,   )

         snake_case											=							output.images
         snake_case											=							image[0, -3:, -3:, -1]
         snake_case											=							[0.2383, 0.2276, 0.236, 0.2192, 0.2186, 0.2053, 0.1971, 0.1901, 0.1719]

         assert image.shape == (1, 5_1_2, 5_1_2, 3)

         assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-2
    def 						a_     (			self       ):
         snake_case											=							StableDiffusionPipeline.from_pretrained('''runwayml/stable-diffusion-v1-5'''     ,   safety_checker=__snake_case       )
         snake_case											=							LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config       )
         snake_case											=							sd_pipe.to(__snake_case       )
         sd_pipe.set_progress_bar_config(disable=__snake_case       )

         snake_case											=							'''padme amidala taking a bath artwork, safe for work, no nudity'''
         snake_case											=							2_7_3_4_9_7_1_7_5_5
         snake_case											=							7

         snake_case											=							torch.manual_seed(__snake_case       )
         snake_case											=							sd_pipe(
             [prompt]     ,   generator=__snake_case     ,   guidance_scale=__snake_case     ,   num_inference_steps=5_0     ,   output_type='''np'''     ,   width=5_1_2     ,   height=5_1_2     ,   sld_guidance_scale=0     ,   )

         snake_case											=							output.images
         snake_case											=							image[0, -3:, -3:, -1]
         snake_case											=							[0.3502, 0.3622, 0.3396, 0.3642, 0.3478, 0.3318, 0.35, 0.3348, 0.3297]

         assert image.shape == (1, 5_1_2, 5_1_2, 3)

         assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-2

         snake_case											=							torch.manual_seed(__snake_case       )
         snake_case											=							sd_pipe(
             [prompt]     ,   generator=__snake_case     ,   guidance_scale=__snake_case     ,   num_inference_steps=5_0     ,   output_type='''np'''     ,   width=5_1_2     ,   height=5_1_2     ,   sld_guidance_scale=2_0_0_0     ,   sld_warmup_steps=7     ,   sld_threshold=0.025     ,   sld_momentum_scale=0.5     ,   sld_mom_beta=0.7     ,   )

         snake_case											=							output.images
         snake_case											=							image[0, -3:, -3:, -1]
         snake_case											=							[0.5531, 0.5206, 0.4895, 0.5156, 0.5182, 0.4751, 0.4802, 0.4803, 0.4443]

         assert image.shape == (1, 5_1_2, 5_1_2, 3)

         assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-2




    def 						a_     (			self       ):
         snake_case											=							StableDiffusionPipeline.from_pretrained('''runwayml/stable-diffusion-v1-5'''       )
         snake_case											=							sd_pipe.to(__snake_case       )
         sd_pipe.set_progress_bar_config(disable=__snake_case       )

         snake_case											=							(
             '''the four horsewomen of the apocalypse, painting by tom of finland, gaston bussiere, craig mullins, j. c.'''
             ''' leyendecker'''
         )
         snake_case											=							1_0_4_4_3_5_5_2_3_4
         snake_case											=							1_2

         snake_case											=							torch.manual_seed(__snake_case       )
         snake_case											=							sd_pipe(
             [prompt]     ,   generator=__snake_case     ,   guidance_scale=__snake_case     ,   num_inference_steps=5_0     ,   output_type='''np'''     ,   width=5_1_2     ,   height=5_1_2     ,   sld_guidance_scale=0     ,   )

         snake_case											=							output.images
         snake_case											=							image[0, -3:, -3:, -1]
         snake_case											=							np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]       )

         assert image.shape == (1, 5_1_2, 5_1_2, 3)

         assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-7

         snake_case											=							torch.manual_seed(__snake_case       )
         snake_case											=							sd_pipe(
             [prompt]     ,   generator=__snake_case     ,   guidance_scale=__snake_case     ,   num_inference_steps=5_0     ,   output_type='''np'''     ,   width=5_1_2     ,   height=5_1_2     ,   sld_guidance_scale=2_0_0_0     ,   sld_warmup_steps=7     ,   sld_threshold=0.025     ,   sld_momentum_scale=0.5     ,   sld_mom_beta=0.7     ,   )

         snake_case											=							output.images
         snake_case											=							image[0, -3:, -3:, -1]
         snake_case											=							np.array([0.5818, 0.6285, 0.6835, 0.6019, 0.625, 0.6754, 0.6096, 0.6334, 0.6561]       )
         assert image.shape == (1, 5_1_2, 5_1_2, 3)

         assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-2








import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


_SCREAMING_SNAKE_CASE     :					List[str]      =       get_tests_dir("fixtures/test_sentencepiece.model")

if is_torch_available():
 from transformers.models.mbart.modeling_mbart import shift_tokens_right

_SCREAMING_SNAKE_CASE     :					Any      =       25_00_04
_SCREAMING_SNAKE_CASE     :					Any      =       25_00_20

@require_sentencepiece
@require_tokenizers
class     A__   (       snake_case__			,   unittest.TestCase      ):







    """simple docstring"""

    __magic_name__         =						MBartaaTokenizer
    __magic_name__         =						MBartaaTokenizerFast
    __magic_name__         =						True
    __magic_name__         =						True
    def 						a_     (			self       ):
         super().setUp()

         # We have a SentencePiece fixture for testing
         snake_case											=							MBartaaTokenizer(__snake_case     ,   src_lang='''en_XX'''     ,   tgt_lang='''ro_RO'''     ,   keep_accents=__snake_case       )
         tokenizer.save_pretrained(self.tmpdirname       )
    def 						a_     (			self       ):
         snake_case											=							'''<s>'''
         snake_case											=							0

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case       )     ,   __snake_case       )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case       )     ,   __snake_case       )
    def 						a_     (			self       ):
         snake_case											=							list(self.get_tokenizer().get_vocab().keys()       )

         self.assertEqual(vocab_keys[0]     ,   '''<s>'''       )
         self.assertEqual(vocab_keys[1]     ,   '''<pad>'''       )
         self.assertEqual(vocab_keys[-1]     ,   '''<mask>'''       )
         self.assertEqual(len(__snake_case       )     ,   1_0_5_4       )
    def 						a_     (			self       ):
         self.assertEqual(self.get_tokenizer().vocab_size     ,   1_0_5_4       )
    def 						a_     (			self       ):
         snake_case											=							MBartaaTokenizer(__snake_case     ,   src_lang='''en_XX'''     ,   tgt_lang='''ro_RO'''     ,   keep_accents=__snake_case       )

         snake_case											=							tokenizer.tokenize('''This is a test'''       )
         self.assertListEqual(__snake_case     ,   ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']       )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(__snake_case       )     ,   [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]     ,   )

         snake_case											=							tokenizer.tokenize('''I was born in 92000, and this is falsé.'''       )
         self.assertListEqual(
             __snake_case     ,   [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.''']     ,   )
         snake_case											=							tokenizer.convert_tokens_to_ids(__snake_case       )
         self.assertListEqual(
             __snake_case     ,   [
                 value + tokenizer.fairseq_offset
                 for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
             ]     ,   )

         snake_case											=							tokenizer.convert_ids_to_tokens(__snake_case       )
         self.assertListEqual(
             __snake_case     ,   [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.''']     ,   )
    @slow
    def 						a_     (			self       ):
         # fmt: off
         snake_case											=							{'''input_ids''': [[2_5_0_0_0_4, 1_1_0_6_2, 8_2_7_7_2, 7, 1_5, 8_2_7_7_2, 5_3_8, 5_1_5_2_9, 2_3_7, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 2_1_5_1_7_5, 1_3_1_4, 1_3_6, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 5_6_3_5_9, 4_2, 1_2_2_0_0_9, 9, 1_6_4_6_6, 1_6, 8_7_3_4_4, 4_5_3_7, 9, 4_7_1_7, 7_8_3_8_1, 6, 1_5_9_9_5_8, 7, 1_5, 2_4_4_8_0, 6_1_8, 4, 5_2_7, 2_2_6_9_3, 5_4_2_8, 4, 2_7_7_7, 2_4_4_8_0, 9_8_7_4, 4, 4_3_5_2_3, 5_9_4, 4, 8_0_3, 1_8_3_9_2, 3_3_1_8_9, 1_8, 4, 4_3_5_2_3, 2_4_4_4_7, 1_2_3_9_9, 1_0_0, 2_4_9_5_5, 8_3_6_5_8, 9_6_2_6, 1_4_4_0_5_7, 1_5, 8_3_9, 2_2_3_3_5, 1_6, 1_3_6, 2_4_9_5_5, 8_3_6_5_8, 8_3_4_7_9, 1_5, 3_9_1_0_2, 7_2_4, 1_6, 6_7_8, 6_4_5, 2_7_8_9, 1_3_2_8, 4_5_8_9, 4_2, 1_2_2_0_0_9, 1_1_5_7_7_4, 2_3, 8_0_5, 1_3_2_8, 4_6_8_7_6, 7, 1_3_6, 5_3_8_9_4, 1_9_4_0, 4_2_2_2_7, 4_1_1_5_9, 1_7_7_2_1, 8_2_3, 4_2_5, 4, 2_7_5_1_2, 9_8_7_2_2, 2_0_6, 1_3_6, 5_5_3_1, 4_9_7_0, 9_1_9, 1_7_3_3_6, 5, 2], [2_5_0_0_0_4, 2_0_0_8_0, 6_1_8, 8_3, 8_2_7_7_5, 4_7, 4_7_9, 9, 1_5_1_7, 7_3, 5_3_8_9_4, 3_3_3, 8_0_5_8_1, 1_1_0_1_1_7, 1_8_8_1_1, 5_2_5_6, 1_2_9_5, 5_1, 1_5_2_5_2_6, 2_9_7, 7_9_8_6, 3_9_0, 1_2_4_4_1_6, 5_3_8, 3_5_4_3_1, 2_1_4, 9_8, 1_5_0_4_4, 2_5_7_3_7, 1_3_6, 7_1_0_8, 4_3_7_0_1, 2_3, 7_5_6, 1_3_5_3_5_5, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_5_0_0_0_4, 5_8_1, 6_3_7_7_3, 1_1_9_4_5_5, 6, 1_4_7_7_9_7, 8_8_2_0_3, 7, 6_4_5, 7_0, 2_1, 3_2_8_5, 1_0_2_6_9, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=__snake_case     ,   model_name='''facebook/mbart-large-50'''     ,   revision='''d3913889c59cd5c9e456b269c376325eabad57e2'''     ,   )




    def 						a_     (			self       ):
         if not self.test_slow_tokenizer:
              # as we don't have a slow version, we can't compare the outputs between slow and fast versions
              return

         snake_case											=							(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-mbart50''', {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                   snake_case											=							self.rust_tokenizer_class.from_pretrained(__snake_case     ,   **__snake_case       )
                   snake_case											=							self.tokenizer_class.from_pretrained(__snake_case     ,   **__snake_case       )

                   snake_case											=							tempfile.mkdtemp()

                   snake_case											=							tokenizer_r.save_pretrained(__snake_case       )
                   snake_case											=							tokenizer_p.save_pretrained(__snake_case       )

                   # Checks it save with the same files + the tokenizer.json file for the fast one
                   self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files       )       )
                   snake_case											=							tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f       )
                   self.assertSequenceEqual(__snake_case     ,   __snake_case       )

                   # Checks everything loads correctly in the same way
                   snake_case											=							tokenizer_r.from_pretrained(__snake_case       )
                   snake_case											=							tokenizer_p.from_pretrained(__snake_case       )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                        self.assertTrue(hasattr(__snake_case     ,   __snake_case       )       )
                        # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
                        # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

                   shutil.rmtree(__snake_case       )

                   # Save tokenizer rust, legacy_format=True
                   snake_case											=							tempfile.mkdtemp()

                   snake_case											=							tokenizer_r.save_pretrained(__snake_case     ,   legacy_format=__snake_case       )
                   snake_case											=							tokenizer_p.save_pretrained(__snake_case       )

                   # Checks it save with the same files
                   self.assertSequenceEqual(__snake_case     ,   __snake_case       )

                   # Checks everything loads correctly in the same way
                   snake_case											=							tokenizer_r.from_pretrained(__snake_case       )
                   snake_case											=							tokenizer_p.from_pretrained(__snake_case       )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                        self.assertTrue(hasattr(__snake_case     ,   __snake_case       )       )

                   shutil.rmtree(__snake_case       )

                   # Save tokenizer rust, legacy_format=False
                   snake_case											=							tempfile.mkdtemp()

                   snake_case											=							tokenizer_r.save_pretrained(__snake_case     ,   legacy_format=__snake_case       )
                   snake_case											=							tokenizer_p.save_pretrained(__snake_case       )

                   # Checks it saved the tokenizer.json file
                   self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files       )       )

                   # Checks everything loads correctly in the same way
                   snake_case											=							tokenizer_r.from_pretrained(__snake_case       )
                   snake_case											=							tokenizer_p.from_pretrained(__snake_case       )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                        self.assertTrue(hasattr(__snake_case     ,   __snake_case       )       )

                   shutil.rmtree(__snake_case       )






@require_torch
@require_sentencepiece
@require_tokenizers
class     A__   (       unittest.TestCase      ):







    """simple docstring"""

    __magic_name__         =						'facebook/mbart-large-50-one-to-many-mmt'
    __magic_name__         =						[
        ' UN Chief Says There Is No Military Solution in Syria',
        ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
    ]
    __magic_name__         =						[
        'Şeful ONU declară că nu există o soluţie militară în Siria',
        'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
        ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
        ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
    ]
    __magic_name__         =						[EN_CODE, 82_74, 12_78_73, 2_59_16, 7, 86_22, 20_71, 4_38, 6_74_85, 53, 18_78_95, 23, 5_17_12, 2]
    @classmethod
    def 						a_     (			cls       ):
         snake_case											=							MBartaaTokenizer.from_pretrained(
             cls.checkpoint_name     ,   src_lang='''en_XX'''     ,   tgt_lang='''ro_RO'''       )
         snake_case											=							1
         return cls
    def 						a_     (			self       ):
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ar_AR''']     ,   2_5_0_0_0_1       )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''en_EN''']     ,   2_5_0_0_0_4       )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ro_RO''']     ,   2_5_0_0_2_0       )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''mr_IN''']     ,   2_5_0_0_3_8       )
    def 						a_     (			self       ):
         snake_case											=							self.tokenizer.batch_encode_plus(self.src_text       ).input_ids[0]
         self.assertListEqual(self.expected_src_tokens     ,   __snake_case       )
    def 						a_     (			self       ):
         self.assertIn(__snake_case     ,   self.tokenizer.all_special_ids       )
         snake_case											=							[RO_CODE, 8_8_4, 9_0_1_9, 9_6, 9, 9_1_6, 8_6_7_9_2, 3_6, 1_8_7_4_3, 1_5_5_9_6, 5, 2]
         snake_case											=							self.tokenizer.decode(__snake_case     ,   skip_special_tokens=__snake_case       )
         snake_case											=							self.tokenizer.decode(generated_ids[1:]     ,   skip_special_tokens=__snake_case       )
         self.assertEqual(__snake_case     ,   __snake_case       )
         self.assertNotIn(self.tokenizer.eos_token     ,   __snake_case       )
    def 						a_     (			self       ):
         snake_case											=							['''this is gunna be a long sentence ''' * 2_0]
         assert isinstance(src_text[0]     ,   __snake_case       )
         snake_case											=							1_0
         snake_case											=							self.tokenizer(__snake_case     ,   max_length=__snake_case     ,   truncation=__snake_case       ).input_ids[0]
         self.assertEqual(ids[0]     ,   __snake_case       )
         self.assertEqual(ids[-1]     ,   2       )
         self.assertEqual(len(__snake_case       )     ,   __snake_case       )
    def 						a_     (			self       ):
         self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']       )     ,   [2_5_0_0_5_3, 2_5_0_0_0_1]       )
    def 						a_     (			self       ):
         snake_case											=							tempfile.mkdtemp()
         snake_case											=							self.tokenizer.fairseq_tokens_to_ids
         self.tokenizer.save_pretrained(__snake_case       )
         snake_case											=							MBartaaTokenizer.from_pretrained(__snake_case       )
         self.assertDictEqual(new_tok.fairseq_tokens_to_ids     ,   __snake_case       )
    @require_torch
    def 						a_     (			self       ):
         snake_case											=							self.tokenizer(self.src_text     ,   text_target=self.tgt_text     ,   padding=__snake_case     ,   return_tensors='''pt'''       )
         snake_case											=							shift_tokens_right(batch['''labels''']     ,   self.tokenizer.pad_token_id       )

         # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
         assert batch.input_ids[1][0] == EN_CODE
         assert batch.input_ids[1][-1] == 2
         assert batch.labels[1][0] == RO_CODE
         assert batch.labels[1][-1] == 2
         assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]
    @require_torch
    def 						a_     (			self       ):
         snake_case											=							self.tokenizer(
             self.src_text     ,   text_target=self.tgt_text     ,   padding=__snake_case     ,   truncation=__snake_case     ,   max_length=len(self.expected_src_tokens       )     ,   return_tensors='''pt'''     ,   )
         snake_case											=							shift_tokens_right(batch['''labels''']     ,   self.tokenizer.pad_token_id       )

         self.assertIsInstance(__snake_case     ,   __snake_case       )

         self.assertEqual((2, 1_4)     ,   batch.input_ids.shape       )
         self.assertEqual((2, 1_4)     ,   batch.attention_mask.shape       )
         snake_case											=							batch.input_ids.tolist()[0]
         self.assertListEqual(self.expected_src_tokens     ,   __snake_case       )
         self.assertEqual(2     ,   batch.decoder_input_ids[0, 0]       )  # decoder_start_token_id
         # Test that special tokens are reset
         self.assertEqual(self.tokenizer.prefix_tokens     ,   [EN_CODE]       )
         self.assertEqual(self.tokenizer.suffix_tokens     ,   [self.tokenizer.eos_token_id]       )
    def 						a_     (			self       ):
         snake_case											=							self.tokenizer(self.src_text     ,   padding=__snake_case     ,   truncation=__snake_case     ,   max_length=3     ,   return_tensors='''pt'''       )
         snake_case											=							self.tokenizer(
             text_target=self.tgt_text     ,   padding=__snake_case     ,   truncation=__snake_case     ,   max_length=1_0     ,   return_tensors='''pt'''       )
         snake_case											=							targets['''input_ids''']
         snake_case											=							shift_tokens_right(__snake_case     ,   self.tokenizer.pad_token_id       )

         self.assertEqual(batch.input_ids.shape[1]     ,   3       )
         self.assertEqual(batch.decoder_input_ids.shape[1]     ,   1_0       )




    @require_torch
    def 						a_     (			self       ):
         snake_case											=							self.tokenizer._build_translation_inputs(
             '''A test'''     ,   return_tensors='''pt'''     ,   src_lang='''en_XX'''     ,   tgt_lang='''ar_AR'''       )

         self.assertEqual(
             nested_simplify(__snake_case       )     ,   {
                 # en_XX, A, test, EOS
                 '''input_ids''': [[2_5_0_0_0_4, 6_2, 3_0_3_4, 2]],
                 '''attention_mask''': [[1, 1, 1, 1]],
                 # ar_AR
                 '''forced_bos_token_id''': 2_5_0_0_0_1,
             }     ,   )




