




'''simple docstring'''



import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 			lowerCAmelCase      (       __lowerCAmelCase ,      unittest.TestCase):
       __lowercase     :					Tuple       				=	GPTNeoXJapaneseTokenizer
       __lowercase     :					Optional[Any]       				=	False
       __lowercase     :					Tuple       				=	{'''do_clean_text''': False, '''add_prefix_space''': False}


       def 					lowerCAmelCase					(      self     )					->     int:



            '''simple docstring'''




            super().setUp()

            __snake_case			    =     [
                '''こん''',
                '''こんに''',
                '''にちは''',
                '''ばんは''',
                '''世界,㔺界''',
                '''、''',
                '''。''',
                '''<BR>''',
                '''<SP>''',
                '''<TAB>''',
                '''<URL>''',
                '''<EMAIL>''',
                '''<TEL>''',
                '''<DATE>''',
                '''<PRICE>''',
                '''<BLOCK>''',
                '''<KIGOU>''',
                '''<U2000U2BFF>''',
                '''<|emoji1|>''',
                '''<unk>''',
                '''<|startoftext|>''',
                '''<|endoftext|>''',
            ]
            __snake_case			    =     {'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
            __snake_case			    =     {'''unk_token''': '''<unk>'''}

            __snake_case			    =     os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES['''vocab_file''']     )
            __snake_case			    =     os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES['''emoji_file''']     )
            with open(self.vocab_file			,     '''w'''			,     encoding='''utf-8'''     ) as vocab_writer:
                 vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]     )     )
            with open(self.emoji_file			,     '''w'''     ) as emoji_writer:
                 emoji_writer.write(json.dumps(__SCREAMING_SNAKE_CASE     )     )


       def 					lowerCAmelCase					(      self			,     **__SCREAMING_SNAKE_CASE     )					->     List[str]:



            '''simple docstring'''




            kwargs.update(self.special_tokens_map     )
            return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname			,     **__SCREAMING_SNAKE_CASE     )


       def 					lowerCAmelCase					(      self			,     __SCREAMING_SNAKE_CASE     )					->     List[Any]:



            '''simple docstring'''




            __snake_case			    =     '''こんにちは、世界。 \nこんばんは、㔺界。😀'''
            __snake_case			    =     '''こんにちは、世界。 \nこんばんは、世界。😀'''
            return input_text, output_text


       def 					lowerCAmelCase					(      self			,     __SCREAMING_SNAKE_CASE     )					->     List[Any]:



            '''simple docstring'''




            __snake_case    ,			__snake_case			    =     self.get_input_output_texts(__SCREAMING_SNAKE_CASE     )
            __snake_case			    =     tokenizer.encode(__SCREAMING_SNAKE_CASE			,     add_special_tokens=__SCREAMING_SNAKE_CASE     )
            __snake_case			    =     tokenizer.decode(__SCREAMING_SNAKE_CASE			,     clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE     )
            return text, ids


       def 					lowerCAmelCase					(      self     )					->     str:



            '''simple docstring'''




            pass  # TODO add if relevant


       def 					lowerCAmelCase					(      self     )					->     Optional[Any]:



            '''simple docstring'''




            pass  # TODO add if relevant


       def 					lowerCAmelCase					(      self     )					->     Optional[int]:



            '''simple docstring'''




            pass  # TODO add if relevant


       def 					lowerCAmelCase					(      self     )					->     Tuple:



            '''simple docstring'''




            __snake_case			    =     self.get_tokenizer()

            # Testing tokenization
            __snake_case			    =     '''こんにちは、世界。　こんばんは、㔺界。'''
            __snake_case			    =     ['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
            __snake_case			    =     tokenizer.tokenize(__SCREAMING_SNAKE_CASE     )
            self.assertListEqual(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     )

            # Testing conversion to ids without special tokens
            __snake_case			    =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
            __snake_case			    =     tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE     )
            self.assertListEqual(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     )

            # Testing conversion to ids with special tokens
            __snake_case			    =     tokens + [tokenizer.unk_token]
            __snake_case			    =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
            __snake_case			    =     tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE     )
            self.assertListEqual(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     )


       @slow
       def 					lowerCAmelCase					(      self     )					->     str:



            '''simple docstring'''




            __snake_case			    =     self.tokenizer_class.from_pretrained('''abeja/gpt-neox-japanese-2.7b'''     )

            __snake_case			    =     tokenizer.encode('''ありがとう。'''			,     add_special_tokens=__SCREAMING_SNAKE_CASE     )
            __snake_case			    =     tokenizer.encode('''どういたしまして。'''			,     add_special_tokens=__SCREAMING_SNAKE_CASE     )

            __snake_case			    =     tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE     )
            __snake_case			    =     tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     )

            assert encoded_sentence == ids_a
            assert encoded_pair == ids_a + ids_a


       def 					lowerCAmelCase					(      self     )					->     List[Any]:



            '''simple docstring'''




            pass




       def 					lowerCAmelCase					(      self     )					->     Optional[int]:



            '''simple docstring'''




            pass












'''simple docstring'''



import os

from packaging import version

from .. import __version__
from .accelerate_utils import apply_forward_hook
from .constants import (
    CONFIG_NAME,
    DEPRECATED_REVISION_ARGS,
    DIFFUSERS_CACHE,
    DIFFUSERS_DYNAMIC_MODULE_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    ONNX_EXTERNAL_WEIGHTS_NAME,
    ONNX_WEIGHTS_NAME,
    SAFETENSORS_WEIGHTS_NAME,
    TEXT_ENCODER_ATTN_MODULE,
    WEIGHTS_NAME,
)
from .deprecation_utils import deprecate
from .doc_utils import replace_example_docstring
from .dynamic_modules_utils import get_class_from_dynamic_module
from .hub_utils import (
    HF_HUB_OFFLINE,
    _add_variant,
    _get_model_file,
    extract_commit_hash,
    http_user_agent,
)
from .import_utils import (
    BACKENDS_MAPPING,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    DummyObject,
    OptionalDependencyNotAvailable,
    is_accelerate_available,
    is_accelerate_version,
    is_bsa_available,
    is_flax_available,
    is_ftfy_available,
    is_inflect_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_k_diffusion_version,
    is_librosa_available,
    is_note_seq_available,
    is_omegaconf_available,
    is_onnx_available,
    is_safetensors_available,
    is_scipy_available,
    is_tensorboard_available,
    is_tf_available,
    is_torch_available,
    is_torch_version,
    is_torchsde_available,
    is_transformers_available,
    is_transformers_version,
    is_unidecode_available,
    is_wandb_available,
    is_xformers_available,
    requires_backends,
)
from .logging import get_logger
from .outputs import BaseOutput
from .pil_utils import PIL_INTERPOLATION, numpy_to_pil, pt_to_pil
from .torch_utils import is_compiled_module, randn_tensor


if is_torch_available():
     from .testing_utils import (
         floats_tensor,
         load_hf_numpy,
         load_image,
         load_numpy,
         load_pt,
         nightly,
         parse_flag_from_env,
         print_tensor_test,
         require_torch_a,
         require_torch_gpu,
         skip_mps,
         slow,
         torch_all_close,
         torch_device,
     )
     from .torch_utils import maybe_allow_in_graph

from .testing_utils import export_to_gif, export_to_video


UpperCAmelCase_				:     int     			=						get_logger(__name__)







def 	_UpperCamelCase       (_lowerCamelCase							:   Union[str, Any]		)->      Tuple:




     '''simple docstring'''





     if version.parse(_lowerCamelCase		) < version.parse(_lowerCamelCase		):
          if "dev" in min_version:
               __snake_case			    =     (
                   '''This example requires a source install from HuggingFace diffusers (see '''
                   '''`https://huggingface.co/docs/diffusers/installation#install-from-source`),'''
               )
          else:
               __snake_case			    =     f'''This example requires a minimum version of {min_version},'''
          error_message += f''' but the version found is {__version__}.\n'''
          raise ImportError(_lowerCamelCase		)






