




"""simple docstring"""

import gc
import unittest

import torch
from parameterized import parameterized

from diffusers import AutoencoderKL
from diffusers.utils import floats_tensor, load_hf_numpy, require_torch_gpu, slow, torch_all_close, torch_device
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


enable_full_determinism()






class        SCREAMING_SNAKE_CASE__   (						snake_case__ ,				snake_case__ ,				unittest.TestCase      ):
   snake_case			   =       AutoencoderKL
   snake_case			   =       '''sample'''
   snake_case			   =       1E-2
   @property
   def    __UpperCAmelCase       (       self			:				Optional[Any]							):
          lowerCamelCase__          =						4
          lowerCamelCase__          =						3
          lowerCamelCase__          =						(32, 32)

          lowerCamelCase__          =						floats_tensor((batch_size, num_channels) + sizes							).to(_UpperCAmelCase							)

          return {"sample": image}
   @property
   def    __UpperCAmelCase       (       self			:				int							):
          return (3, 32, 32)
   @property
   def    __UpperCAmelCase       (       self			:				Any							):
          return (3, 32, 32)
   def    __UpperCAmelCase       (       self			:				List[Any]							):
          lowerCamelCase__          =						{
              """block_out_channels""": [32, 64],
              """in_channels""": 3,
              """out_channels""": 3,
              """down_block_types""": ["""DownEncoderBlock2D""", """DownEncoderBlock2D"""],
              """up_block_types""": ["""UpDecoderBlock2D""", """UpDecoderBlock2D"""],
              """latent_channels""": 4,
          }
          lowerCamelCase__          =						self.dummy_input
          return init_dict, inputs_dict
   def    __UpperCAmelCase       (       self			:				List[str]							):
          pass
   def    __UpperCAmelCase       (       self			:				Tuple							):
          pass
   @unittest.skipIf(torch_device == """mps"""   ,	"""Gradient checkpointing skipped on MPS"""							)
   def    __UpperCAmelCase       (       self			:				List[str]							):
          # enable deterministic behavior for gradient checkpointing
          lowerCamelCase__     ,	lowerCamelCase__          =						self.prepare_init_args_and_inputs_for_common()
          lowerCamelCase__          =						self.model_class(**_UpperCAmelCase							)
          model.to(_UpperCAmelCase							)

          assert not model.is_gradient_checkpointing and model.training

          lowerCamelCase__          =						model(**_UpperCAmelCase							).sample
          # run the backwards pass on the model. For backwards pass, for simplicity purpose,
          # we won't calculate the loss and rather backprop on out.sum()
          model.zero_grad()

          lowerCamelCase__          =						torch.randn_like(_UpperCAmelCase							)
          lowerCamelCase__          =						(out - labels).mean()
          loss.backward()

          # re-instantiate the model now enabling gradient checkpointing
          lowerCamelCase__          =						self.model_class(**_UpperCAmelCase							)
          # clone model
          model_a.load_state_dict(model.state_dict()							)
          model_a.to(_UpperCAmelCase							)
          model_a.enable_gradient_checkpointing()

          assert model_a.is_gradient_checkpointing and model_a.training

          lowerCamelCase__          =						model_a(**_UpperCAmelCase							).sample
          # run the backwards pass on the model. For backwards pass, for simplicity purpose,
          # we won't calculate the loss and rather backprop on out.sum()
          model_a.zero_grad()
          lowerCamelCase__          =						(out_a - labels).mean()
          loss_a.backward()

          # compare the output and parameters gradients
          self.assertTrue((loss - loss_a).abs() < 1e-5							)
          lowerCamelCase__          =						dict(model.named_parameters()							)
          lowerCamelCase__          =						dict(model_a.named_parameters()							)
          for name, param in named_params.items():
                 self.assertTrue(torch_all_close(param.grad.data   ,	named_params_a[name].grad.data   ,	atol=5e-5							)							)
   def    __UpperCAmelCase       (       self			:				Optional[int]							):
          lowerCamelCase__     ,	lowerCamelCase__          =						AutoencoderKL.from_pretrained("""fusing/autoencoder-kl-dummy"""   ,	output_loading_info=_UpperCAmelCase							)
          self.assertIsNotNone(_UpperCAmelCase							)
          self.assertEqual(len(loading_info["""missing_keys"""]							)   ,	0							)

          model.to(_UpperCAmelCase							)
          lowerCamelCase__          =						model(**self.dummy_input							)

          assert image is not None, "Make sure output is not None"





   def    __UpperCAmelCase       (       self			:				Tuple							):
          lowerCamelCase__          =						AutoencoderKL.from_pretrained("""fusing/autoencoder-kl-dummy"""							)
          lowerCamelCase__          =						model.to(_UpperCAmelCase							)
          model.eval()

          if torch_device == "mps":
                 lowerCamelCase__          =						torch.manual_seed(0							)
          else:
                 lowerCamelCase__          =						torch.Generator(device=_UpperCAmelCase							).manual_seed(0							)

          lowerCamelCase__          =						torch.randn(
              1   ,	model.config.in_channels   ,	model.config.sample_size   ,	model.config.sample_size   ,	generator=torch.manual_seed(0							)   ,	)
          lowerCamelCase__          =						image.to(_UpperCAmelCase							)
          with torch.no_grad():
                 lowerCamelCase__          =						model(_UpperCAmelCase   ,	sample_posterior=_UpperCAmelCase   ,	generator=_UpperCAmelCase							).sample

          lowerCamelCase__          =						output[0, -1, -3:, -3:].flatten().cpu()

          # Since the VAE Gaussian prior's generator is seeded on the appropriate device,
          # the expected output slices are not the same for CPU and GPU.
          if torch_device == "mps":
                 lowerCamelCase__          =						torch.tensor(
                     [
                         -4.0_078e-01,
                         -3.8_323e-04,
                         -1.2_681e-01,
                         -1.1_462e-01,
                         2.0_095e-01,
                         1.0_893e-01,
                         -8.8_247e-02,
                         -3.0_361e-01,
                         -9.8_644e-03,
                     ]							)
          elif torch_device == "cpu":
                 lowerCamelCase__          =						torch.tensor(
                     [-0.1_3_5_2, 0.0_8_7_8, 0.0_4_1_9, -0.0_8_1_8, -0.1_0_6_9, 0.0_6_8_8, -0.1_4_5_8, -0.4_4_4_6, -0.0_0_2_6]							)
          else:
                 lowerCamelCase__          =						torch.tensor(
                     [-0.2_4_2_1, 0.4_6_4_2, 0.2_5_0_7, -0.0_4_3_8, 0.0_6_8_2, 0.3_1_6_0, -0.2_0_1_8, -0.0_7_2_7, 0.2_4_8_5]							)

          self.assertTrue(torch_all_close(_UpperCAmelCase   ,	_UpperCAmelCase   ,	rtol=1e-2							)							)






@slow
class        SCREAMING_SNAKE_CASE__   (						unittest.TestCase      ):
   def    __UpperCAmelCase       (       self			:				int   ,	SCREAMING_SNAKE_CASE_			:				Union[str, Any]   ,	SCREAMING_SNAKE_CASE_			:				Tuple							):
          return f"""gaussian_noise_s={seed}_shape={'_'.join([str(_UpperCAmelCase							) for s in shape]							)}.npy"""
   def    __UpperCAmelCase       (       self			:				Tuple							):
          # clean up the VRAM after each test
          super().tearDown()
          gc.collect()
          torch.cuda.empty_cache()
   def    __UpperCAmelCase       (       self			:				Optional[int]   ,	SCREAMING_SNAKE_CASE_			:				Optional[int]=0   ,	SCREAMING_SNAKE_CASE_			:				Dict=(4, 3, 512, 512)   ,	SCREAMING_SNAKE_CASE_			:				Tuple=False							):
          lowerCamelCase__          =						torch.floataa if fpaa else torch.floataa
          lowerCamelCase__          =						torch.from_numpy(load_hf_numpy(self.get_file_format(_UpperCAmelCase   ,	_UpperCAmelCase							)							)							).to(_UpperCAmelCase							).to(_UpperCAmelCase							)
          return image
   def    __UpperCAmelCase       (       self			:				Tuple   ,	SCREAMING_SNAKE_CASE_			:				str="CompVis/stable-diffusion-v1-4"   ,	SCREAMING_SNAKE_CASE_			:				Optional[Any]=False							):
          lowerCamelCase__          =						"""fp16""" if fpaa else None
          lowerCamelCase__          =						torch.floataa if fpaa else torch.floataa

          lowerCamelCase__          =						AutoencoderKL.from_pretrained(
              _UpperCAmelCase   ,	subfolder="""vae"""   ,	torch_dtype=_UpperCAmelCase   ,	revision=_UpperCAmelCase   ,	)
          model.to(_UpperCAmelCase							).eval()

          return model
   def    __UpperCAmelCase       (       self			:				int   ,	SCREAMING_SNAKE_CASE_			:				Optional[int]=0							):
          if torch_device == "mps":
                 return torch.manual_seed(_UpperCAmelCase							)
          return torch.Generator(device=_UpperCAmelCase							).manual_seed(_UpperCAmelCase							)
   @parameterized.expand(
       [
           # fmt: off
           [33, [-0.1_6_0_3, 0.9_8_7_8, -0.0_4_9_5, -0.0_7_9_0, -0.2_7_0_9, 0.8_3_7_5, -0.2_0_6_0, -0.0_8_2_4], [-0.2_3_9_5, 0.0_0_9_8, 0.0_1_0_2, -0.0_7_0_9, -0.2_8_4_0, -0.0_2_7_4, -0.0_7_1_8, -0.1_8_2_4]],
           [47, [-0.2_3_7_6, 0.1_1_6_8, 0.1_3_3_2, -0.4_8_4_0, -0.2_5_0_8, -0.0_7_9_1, -0.0_4_9_3, -0.4_0_8_9], [0.0_3_5_0, 0.0_8_4_7, 0.0_4_6_7, 0.0_3_4_4, -0.0_8_4_2, -0.0_5_4_7, -0.0_6_3_3, -0.1_1_3_1]],
           # fmt: on
       ]							)
   def    __UpperCAmelCase       (       self			:				List[Any]   ,	SCREAMING_SNAKE_CASE_			:				Tuple   ,	SCREAMING_SNAKE_CASE_			:				Union[str, Any]   ,	SCREAMING_SNAKE_CASE_			:				List[str]							):
          lowerCamelCase__          =						self.get_sd_vae_model()
          lowerCamelCase__          =						self.get_sd_image(_UpperCAmelCase							)
          lowerCamelCase__          =						self.get_generator(_UpperCAmelCase							)

          with torch.no_grad():
                 lowerCamelCase__          =						model(_UpperCAmelCase   ,	generator=_UpperCAmelCase   ,	sample_posterior=_UpperCAmelCase							).sample

          assert sample.shape == image.shape

          lowerCamelCase__          =						sample[-1, -2:, -2:, :2].flatten().float().cpu()
          lowerCamelCase__          =						torch.tensor(expected_slice_mps if torch_device == """mps""" else expected_slice							)

          assert torch_all_close(_UpperCAmelCase   ,	_UpperCAmelCase   ,	atol=3e-3							)
   @parameterized.expand(
       [
           # fmt: off
           [33, [-0.0_5_1_3, 0.0_2_8_9, 1.3_7_9_9, 0.2_1_6_6, -0.2_5_7_3, -0.0_8_7_1, 0.5_1_0_3, -0.0_9_9_9]],
           [47, [-0.4_1_2_8, -0.1_3_2_0, -0.3_7_0_4, 0.1_9_6_5, -0.4_1_1_6, -0.2_3_3_2, -0.3_3_4_0, 0.2_2_4_7]],
           # fmt: on
       ]							)
   @require_torch_gpu
   def    __UpperCAmelCase       (       self			:				List[Any]   ,	SCREAMING_SNAKE_CASE_			:				Any   ,	SCREAMING_SNAKE_CASE_			:				Union[str, Any]							):
          lowerCamelCase__          =						self.get_sd_vae_model(fpaa=_UpperCAmelCase							)
          lowerCamelCase__          =						self.get_sd_image(_UpperCAmelCase   ,	fpaa=_UpperCAmelCase							)
          lowerCamelCase__          =						self.get_generator(_UpperCAmelCase							)

          with torch.no_grad():
                 lowerCamelCase__          =						model(_UpperCAmelCase   ,	generator=_UpperCAmelCase   ,	sample_posterior=_UpperCAmelCase							).sample

          assert sample.shape == image.shape

          lowerCamelCase__          =						sample[-1, -2:, :2, -2:].flatten().float().cpu()
          lowerCamelCase__          =						torch.tensor(_UpperCAmelCase							)

          assert torch_all_close(_UpperCAmelCase   ,	_UpperCAmelCase   ,	atol=1e-2							)
   @parameterized.expand(
       [
           # fmt: off
           [33, [-0.1_6_0_9, 0.9_8_6_6, -0.0_4_8_7, -0.0_7_7_7, -0.2_7_1_6, 0.8_3_6_8, -0.2_0_5_5, -0.0_8_1_4], [-0.2_3_9_5, 0.0_0_9_8, 0.0_1_0_2, -0.0_7_0_9, -0.2_8_4_0, -0.0_2_7_4, -0.0_7_1_8, -0.1_8_2_4]],
           [47, [-0.2_3_7_7, 0.1_1_4_7, 0.1_3_3_3, -0.4_8_4_1, -0.2_5_0_6, -0.0_8_0_5, -0.0_4_9_1, -0.4_0_8_5], [0.0_3_5_0, 0.0_8_4_7, 0.0_4_6_7, 0.0_3_4_4, -0.0_8_4_2, -0.0_5_4_7, -0.0_6_3_3, -0.1_1_3_1]],
           # fmt: on
       ]							)
   def    __UpperCAmelCase       (       self			:				List[Any]   ,	SCREAMING_SNAKE_CASE_			:				Optional[int]   ,	SCREAMING_SNAKE_CASE_			:				Any   ,	SCREAMING_SNAKE_CASE_			:				Optional[Any]							):
          lowerCamelCase__          =						self.get_sd_vae_model()
          lowerCamelCase__          =						self.get_sd_image(_UpperCAmelCase							)

          with torch.no_grad():
                 lowerCamelCase__          =						model(_UpperCAmelCase							).sample

          assert sample.shape == image.shape

          lowerCamelCase__          =						sample[-1, -2:, -2:, :2].flatten().float().cpu()
          lowerCamelCase__          =						torch.tensor(expected_slice_mps if torch_device == """mps""" else expected_slice							)

          assert torch_all_close(_UpperCAmelCase   ,	_UpperCAmelCase   ,	atol=3e-3							)
   @parameterized.expand(
       [
           # fmt: off
           [13, [-0.2_0_5_1, -0.1_8_0_3, -0.2_3_1_1, -0.2_1_1_4, -0.3_2_9_2, -0.3_5_7_4, -0.2_9_5_3, -0.3_3_2_3]],
           [37, [-0.2_6_3_2, -0.2_6_2_5, -0.2_1_9_9, -0.2_7_4_1, -0.4_5_3_9, -0.4_9_9_0, -0.3_7_2_0, -0.4_9_2_5]],
           # fmt: on
       ]							)
   @require_torch_gpu
   def    __UpperCAmelCase       (       self			:				int   ,	SCREAMING_SNAKE_CASE_			:				int   ,	SCREAMING_SNAKE_CASE_			:				Any							):
          lowerCamelCase__          =						self.get_sd_vae_model()
          lowerCamelCase__          =						self.get_sd_image(_UpperCAmelCase   ,	shape=(3, 4, 64, 64)							)

          with torch.no_grad():
                 lowerCamelCase__          =						model.decode(_UpperCAmelCase							).sample

          assert list(sample.shape							) == [3, 3, 512, 512]

          lowerCamelCase__          =						sample[-1, -2:, :2, -2:].flatten().cpu()
          lowerCamelCase__          =						torch.tensor(_UpperCAmelCase							)

          assert torch_all_close(_UpperCAmelCase   ,	_UpperCAmelCase   ,	atol=1e-3							)
   @parameterized.expand(
       [
           # fmt: off
           [27, [-0.0_3_6_9, 0.0_2_0_7, -0.0_7_7_6, -0.0_6_8_2, -0.1_7_4_7, -0.1_9_3_0, -0.1_4_6_5, -0.2_0_3_9]],
           [16, [-0.1_6_2_8, -0.2_1_3_4, -0.2_7_4_7, -0.2_6_4_2, -0.3_7_7_4, -0.4_4_0_4, -0.3_6_8_7, -0.4_2_7_7]],
           # fmt: on
       ]							)
   @require_torch_gpu
   def    __UpperCAmelCase       (       self			:				int   ,	SCREAMING_SNAKE_CASE_			:				int   ,	SCREAMING_SNAKE_CASE_			:				Any							):
          lowerCamelCase__          =						self.get_sd_vae_model(fpaa=_UpperCAmelCase							)
          lowerCamelCase__          =						self.get_sd_image(_UpperCAmelCase   ,	shape=(3, 4, 64, 64)   ,	fpaa=_UpperCAmelCase							)

          with torch.no_grad():
                 lowerCamelCase__          =						model.decode(_UpperCAmelCase							).sample

          assert list(sample.shape							) == [3, 3, 512, 512]

          lowerCamelCase__          =						sample[-1, -2:, :2, -2:].flatten().float().cpu()
          lowerCamelCase__          =						torch.tensor(_UpperCAmelCase							)

          assert torch_all_close(_UpperCAmelCase   ,	_UpperCAmelCase   ,	atol=5e-3							)
   @parameterized.expand([(13,), (16,), (27,)]							)
   @require_torch_gpu
   @unittest.skipIf(not is_xformers_available()   ,	reason="""xformers is not required when using PyTorch 2.0."""							)
   def    __UpperCAmelCase       (       self			:				Dict   ,	SCREAMING_SNAKE_CASE_			:				int							):
          lowerCamelCase__          =						self.get_sd_vae_model(fpaa=_UpperCAmelCase							)
          lowerCamelCase__          =						self.get_sd_image(_UpperCAmelCase   ,	shape=(3, 4, 64, 64)   ,	fpaa=_UpperCAmelCase							)

          with torch.no_grad():
                 lowerCamelCase__          =						model.decode(_UpperCAmelCase							).sample

          model.enable_xformers_memory_efficient_attention()
          with torch.no_grad():
                 lowerCamelCase__          =						model.decode(_UpperCAmelCase							).sample

          assert list(sample.shape							) == [3, 3, 512, 512]

          assert torch_all_close(_UpperCAmelCase   ,	_UpperCAmelCase   ,	atol=1e-1							)
   @parameterized.expand([(13,), (16,), (37,)]							)
   @require_torch_gpu
   @unittest.skipIf(not is_xformers_available()   ,	reason="""xformers is not required when using PyTorch 2.0."""							)
   def    __UpperCAmelCase       (       self			:				Optional[int]   ,	SCREAMING_SNAKE_CASE_			:				Any							):
          lowerCamelCase__          =						self.get_sd_vae_model()
          lowerCamelCase__          =						self.get_sd_image(_UpperCAmelCase   ,	shape=(3, 4, 64, 64)							)

          with torch.no_grad():
                 lowerCamelCase__          =						model.decode(_UpperCAmelCase							).sample

          model.enable_xformers_memory_efficient_attention()
          with torch.no_grad():
                 lowerCamelCase__          =						model.decode(_UpperCAmelCase							).sample

          assert list(sample.shape							) == [3, 3, 512, 512]

          assert torch_all_close(_UpperCAmelCase   ,	_UpperCAmelCase   ,	atol=1e-2							)





   @parameterized.expand(
       [
           # fmt: off
           [33, [-0.3_0_0_1, 0.0_9_1_8, -2.6_9_8_4, -3.9_7_2_0, -3.2_0_9_9, -5.0_3_5_3, 1.7_3_3_8, -0.2_0_6_5, 3.4_2_6_7]],
           [47, [-1.5_0_3_0, -4.3_8_7_1, -6.0_3_5_5, -9.1_1_5_7, -1.6_6_6_1, -2.7_8_5_3, 2.1_6_0_7, -5.0_8_2_3, 2.5_6_3_3]],
           # fmt: on
       ]							)
   def    __UpperCAmelCase       (       self			:				Dict   ,	SCREAMING_SNAKE_CASE_			:				List[Any]   ,	SCREAMING_SNAKE_CASE_			:				Optional[Any]							):
          lowerCamelCase__          =						self.get_sd_vae_model()
          lowerCamelCase__          =						self.get_sd_image(_UpperCAmelCase							)
          lowerCamelCase__          =						self.get_generator(_UpperCAmelCase							)

          with torch.no_grad():
                 lowerCamelCase__          =						model.encode(_UpperCAmelCase							).latent_dist
                 lowerCamelCase__          =						dist.sample(generator=_UpperCAmelCase							)

          assert list(sample.shape							) == [image.shape[0], 4] + [i // 8 for i in image.shape[2:]]

          lowerCamelCase__          =						sample[0, -1, -3:, -3:].flatten().cpu()
          lowerCamelCase__          =						torch.tensor(_UpperCAmelCase							)

          lowerCamelCase__          =						3e-3 if torch_device != """mps""" else 1e-2
          assert torch_all_close(_UpperCAmelCase   ,	_UpperCAmelCase   ,	atol=_UpperCAmelCase							)











'''simple docstring'''

import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class 							a__( snake_case__   ,      unittest.TestCase     ):
			a_	: str									=      BertJapaneseTokenizer
			a_	: List[str]									=      False
			a_	: List[Any]									=      True






			def 						_lowercase  (     self						)							->							Union[str, Any]:
					super().setUp()

					snake_case__	     =[
					    '[UNK]',
					    '[CLS]',
					    '[SEP]',
					    'こんにちは',
					    'こん',
					    'にちは',
					    'ばんは',
					    '##こん',
					    '##にちは',
					    '##ばんは',
					    '世界',
					    '##世界',
					    '、',
					    '##、',
					    '。',
					    '##。',
					]

					snake_case__	     =os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES['vocab_file']						)
					with open(self.vocab_file					,   'w'					,   encoding='utf-8'						) as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							str:
					snake_case__	     ='こんにちは、世界。 \nこんばんは、世界。'
					snake_case__	     ='こんにちは 、 世界 。 こんばんは 、 世界 。'
					return input_text, output_text






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Any:
					snake_case__     ,	snake_case__	     =self.get_input_output_texts(_UpperCAmelCase						)
					snake_case__	     =tokenizer.encode(_UpperCAmelCase					,   add_special_tokens=_UpperCAmelCase						)
					snake_case__	     =tokenizer.decode(_UpperCAmelCase					,   clean_up_tokenization_spaces=_UpperCAmelCase						)
					return text, ids






			def 						_lowercase  (     self						)							->							Dict:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							List[str]:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							str:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer_class(self.vocab_file						)

					snake_case__	     =tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)






			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   word_tokenizer_type='mecab'						)
					self.assertIsNotNone(_UpperCAmelCase						)

					snake_case__	     ='こんにちは、世界。\nこんばんは、世界。'
					snake_case__	     =tokenizer.tokenize(_UpperCAmelCase						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

					snake_case__	     =os.path.join(self.tmpdirname					,   'tokenizer.bin'						)
					with open(_UpperCAmelCase					,   'wb'						) as handle:
							pickle.dump(_UpperCAmelCase					,   _UpperCAmelCase						)

					with open(_UpperCAmelCase					,   'rb'						) as handle:
							snake_case__	     =pickle.load(_UpperCAmelCase						)

					snake_case__	     =tokenizer_new.tokenize(_UpperCAmelCase						)

					self.assertListEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =MecabTokenizer(mecab_dic='ipadic'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							str:
					try:
							snake_case__	     =MecabTokenizer(mecab_dic='unidic_lite'						)
					except ModuleNotFoundError:
							return

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							List[str]:
					try:
							snake_case__	     =MecabTokenizer(mecab_dic='unidic'						)
					except ModuleNotFoundError:
							return

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =MecabTokenizer(do_lower_case=_UpperCAmelCase					,   mecab_dic='ipadic'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							List[Any]:
					try:
							snake_case__	     =MecabTokenizer(
							    do_lower_case=_UpperCAmelCase					,   normalize_text=_UpperCAmelCase					,   mecab_option='-d /usr/local/lib/mecab/dic/jumandic'						)
					except RuntimeError:
							# if dict doesn't exist in the system, previous code raises this error.
							return

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =MecabTokenizer(normalize_text=_UpperCAmelCase					,   mecab_dic='ipadic'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   word_tokenizer_type='sudachi'						)
					self.assertIsNotNone(_UpperCAmelCase						)

					snake_case__	     ='こんにちは、世界。\nこんばんは、世界。'
					snake_case__	     =tokenizer.tokenize(_UpperCAmelCase						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

					snake_case__	     =os.path.join(self.tmpdirname					,   'tokenizer.bin'						)
					with open(_UpperCAmelCase					,   'wb'						) as handle:
							pickle.dump(_UpperCAmelCase					,   _UpperCAmelCase						)

					with open(_UpperCAmelCase					,   'rb'						) as handle:
							snake_case__	     =pickle.load(_UpperCAmelCase						)

					snake_case__	     =tokenizer_new.tokenize(_UpperCAmelCase						)

					self.assertListEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							Union[str, Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'					,   sudachi_split_mode='A'						)

					self.assertListEqual(tokenizer.tokenize('外国人参政権'						)					,   ['外国', '人', '参政', '権']						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'					,   sudachi_split_mode='B'						)

					self.assertListEqual(tokenizer.tokenize('外国人参政権'						)					,   ['外国人', '参政権']						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'					,   sudachi_split_mode='C'						)

					self.assertListEqual(tokenizer.tokenize('外国人参政権'						)					,   ['外国人参政権']						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =SudachiTokenizer(do_lower_case=_UpperCAmelCase					,   sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							Dict:
					snake_case__	     =SudachiTokenizer(normalize_text=_UpperCAmelCase					,   sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							Optional[int]:
					snake_case__	     =SudachiTokenizer(trim_whitespace=_UpperCAmelCase					,   sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   word_tokenizer_type='jumanpp'						)
					self.assertIsNotNone(_UpperCAmelCase						)

					snake_case__	     ='こんにちは、世界。\nこんばんは、世界。'
					snake_case__	     =tokenizer.tokenize(_UpperCAmelCase						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

					snake_case__	     =os.path.join(self.tmpdirname					,   'tokenizer.bin'						)
					with open(_UpperCAmelCase					,   'wb'						) as handle:
							pickle.dump(_UpperCAmelCase					,   _UpperCAmelCase						)

					with open(_UpperCAmelCase					,   'rb'						) as handle:
							snake_case__	     =pickle.load(_UpperCAmelCase						)

					snake_case__	     =tokenizer_new.tokenize(_UpperCAmelCase						)

					self.assertListEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			@require_jumanpp
			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =JumanppTokenizer()

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =JumanppTokenizer(do_lower_case=_UpperCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =JumanppTokenizer(normalize_text=_UpperCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =JumanppTokenizer(trim_whitespace=_UpperCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							str:
					snake_case__	     =JumanppTokenizer()

					self.assertListEqual(
					    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'						)					,   ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']					,   )






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

					snake_case__	     ={}
					for i, token in enumerate(_UpperCAmelCase						):
							snake_case__	     =i
					snake_case__	     =WordpieceTokenizer(vocab=_UpperCAmelCase					,   unk_token='[UNK]'						)

					self.assertListEqual(tokenizer.tokenize(''						)					,   []						)

					self.assertListEqual(tokenizer.tokenize('こんにちは'						)					,   ['こんにちは']						)

					self.assertListEqual(tokenizer.tokenize('こんばんは'						)					,   ['こん', '##ばんは']						)

					self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'						)					,   ['こん', '##ばんは', '[UNK]', 'こんにちは']						)






			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'						)
					snake_case__	     =tokenizer.subword_tokenizer

					snake_case__	     =subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'						)
					self.assertListEqual(_UpperCAmelCase					,   ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']						)

					snake_case__	     =subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'						)
					self.assertListEqual(_UpperCAmelCase					,   ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']						)






			def 						_lowercase  (     self						)							->							Optional[int]:
					snake_case__	     =self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'						)

					snake_case__	     =tokenizer.encode('ありがとう。'					,   add_special_tokens=_UpperCAmelCase						)
					snake_case__	     =tokenizer.encode('どういたしまして。'					,   add_special_tokens=_UpperCAmelCase						)

					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase						)
					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase					,   _UpperCAmelCase						)

					# 2 is for "[CLS]", 3 is for "[SEP]"
					assert encoded_sentence == [2] + text + [3]
					assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 							a__( snake_case__   ,      unittest.TestCase     ):
			a_	: int									=      BertJapaneseTokenizer
			a_	: Optional[Any]									=      False






			def 						_lowercase  (     self						)							->							Any:
					super().setUp()

					snake_case__	     =['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

					snake_case__	     =os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES['vocab_file']						)
					with open(self.vocab_file					,   'w'					,   encoding='utf-8'						) as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)






			def 						_lowercase  (     self					,   **_UpperCAmelCase						)							->							Union[str, Any]:
					return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,   subword_tokenizer_type='character'					,   **_UpperCAmelCase						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Optional[Any]:
					snake_case__	     ='こんにちは、世界。 \nこんばんは、世界。'
					snake_case__	     ='こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
					return input_text, output_text






			def 						_lowercase  (     self						)							->							List[Any]:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Union[str, Any]:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Tuple:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   subword_tokenizer_type='character'						)

					snake_case__	     =tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'						)
					self.assertListEqual(
					    _UpperCAmelCase					,   ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']						)
					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

					snake_case__	     ={}
					for i, token in enumerate(_UpperCAmelCase						):
							snake_case__	     =i
					snake_case__	     =CharacterTokenizer(vocab=_UpperCAmelCase					,   unk_token='[UNK]'						)

					self.assertListEqual(tokenizer.tokenize(''						)					,   []						)

					self.assertListEqual(tokenizer.tokenize('こんにちは'						)					,   ['こ', 'ん', 'に', 'ち', 'は']						)

					self.assertListEqual(tokenizer.tokenize('こんにちほ'						)					,   ['こ', 'ん', 'に', 'ち', '[UNK]']						)






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'						)

					snake_case__	     =tokenizer.encode('ありがとう。'					,   add_special_tokens=_UpperCAmelCase						)
					snake_case__	     =tokenizer.encode('どういたしまして。'					,   add_special_tokens=_UpperCAmelCase						)

					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase						)
					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase					,   _UpperCAmelCase						)

					# 2 is for "[CLS]", 3 is for "[SEP]"
					assert encoded_sentence == [2] + text + [3]
					assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 							a__( unittest.TestCase     ):






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     ='cl-tohoku/bert-base-japanese'
					snake_case__	     =AutoTokenizer.from_pretrained(_UpperCAmelCase						)
					self.assertIsInstance(_UpperCAmelCase					,   _UpperCAmelCase						)





class 							a__( unittest.TestCase     ):






			def 						_lowercase  (     self						)							->							int:
					snake_case__	     ='cl-tohoku/bert-base-japanese'
					with self.assertLogs('transformers'					,   level='WARNING'						) as cm:
							BertTokenizer.from_pretrained(_UpperCAmelCase						)
							self.assertTrue(
							    cm.records[0].message.startswith(
							        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
							        ' is called from.'						)						)
					snake_case__	     ='bert-base-cased'
					with self.assertLogs('transformers'					,   level='WARNING'						) as cm:
							BertJapaneseTokenizer.from_pretrained(_UpperCAmelCase						)
							self.assertTrue(
							    cm.records[0].message.startswith(
							        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
							        ' is called from.'						)						)




