


import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
    from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
    import torch_xla.core.xla_model as xm



def UpperCamelCase_				(				__a      )	->   Dict:
      if is_torch_version("<"			,				"2.0.0"      ) or not hasattr(__a			,				"_dynamo"      ):
            return False
      return isinstance(__a			,				torch._dynamo.eval_frame.OptimizedModule      )



def UpperCamelCase_				(				__a			,				__a = True      )	->   Tuple:
      a__	:  Union[str, Any]     =   (torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

      a__	:  List[Any]     =   is_compiled_module(__a      )
      if is_compiled:
            a__	:  Optional[int]     =   model
            a__	:  List[str]     =   model._orig_mod

      if is_deepspeed_available():
            options += (DeepSpeedEngine,)

      while isinstance(__a			,				__a      ):
            a__	:  int     =   model.module

      if not keep_fpaa_wrapper:
            a__	:  Union[str, Any]     =   getattr(__a			,				"forward"      )
            a__	:  Union[str, Any]     =   model.__dict__.pop("_original_forward"			,				__a      )
            if original_forward is not None:
                  while hasattr(__a			,				"__wrapped__"      ):
                        a__	:  int     =   forward.__wrapped__
                        if forward == original_forward:
                              break
                  a__	:  Any     =   forward
            if getattr(__a			,				"_converted_to_transformer_engine"			,				__a      ):
                  convert_model(__a			,				to_transformer_engine=__a      )

      if is_compiled:
            a__	:  List[str]     =   model
            a__	:  Optional[int]     =   compiled_model

      return model



def UpperCamelCase_				(				)	->   int:
      PartialState().wait_for_everyone()



def UpperCamelCase_				(				__a			,				__a      )	->   int:
      if PartialState().distributed_type == DistributedType.TPU:
            xm.save(__a			,				__a      )
      elif PartialState().local_process_index == 0:
            torch.save(__a			,				__a      )



@contextmanager
def UpperCamelCase_				(				**__a      )	->   Optional[int]:
      for key, value in kwargs.items():
            a__	:  int     =   str(__a      )

      yield

      for key in kwargs:
            if key.upper() in os.environ:
                  del os.environ[key.upper()]



def UpperCamelCase_				(				__a      )	->   Dict:
      if not hasattr(__a			,				"__qualname__"      ) and not hasattr(__a			,				"__name__"      ):
            a__	:  Union[str, Any]     =   getattr(__a			,				"__class__"			,				__a      )
      if hasattr(__a			,				"__qualname__"      ):
            return obj.__qualname__
      if hasattr(__a			,				"__name__"      ):
            return obj.__name__
      return str(__a      )



def UpperCamelCase_				(				__a			,				__a      )	->   str:
      for key, value in source.items():
            if isinstance(__a			,				__a      ):
                  a__	:  Any     =   destination.setdefault(__a			,				{}      )
                  merge_dicts(__a			,				__a      )
            else:
                  a__	:  List[str]     =   value

      return destination



def UpperCamelCase_				(				__a = None      )	->   bool:
      if port is None:
            a__	:  int     =   29_500
      with socket.socket(socket.AF_INET			,				socket.SOCK_STREAM      ) as s:
            return s.connect_ex(("localhost", port)      ) == 0








import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


_A					:				Dict				=			get_tests_dir("""fixtures/test_sentencepiece_bpe.model""")



class 					__snake_case	(  __SCREAMING_SNAKE_CASE			,     unittest.TestCase    ):


    '''simple docstring'''







    lowerCamelCase__    :		Any				=	BartphoTokenizer
    lowerCamelCase__    :		Tuple				=	False
    lowerCamelCase__    :		Dict				=	True



    def  lowercase_					(       self     ):





        '''simple docstring'''

        super().setUp()

        SCREAMING_SNAKE_CASE__											=			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']
        SCREAMING_SNAKE_CASE__											=			dict(zip(A_   ,				range(len(A_     )     )     )     )
        SCREAMING_SNAKE_CASE__											=			{'''unk_token''': '''<unk>'''}

        SCREAMING_SNAKE_CASE__											=			os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''monolingual_vocab_file''']     )
        with open(self.monolingual_vocab_file   ,				'''w'''   ,				encoding='''utf-8'''     ) as fp:
            for token in vocab_tokens:
                fp.write(f'''{token} {vocab_tokens[token]}\n'''     )

        SCREAMING_SNAKE_CASE__											=			BartphoTokenizer(A_   ,				self.monolingual_vocab_file   ,				**self.special_tokens_map     )
        tokenizer.save_pretrained(self.tmpdirname     )



    def  lowercase_					(       self   ,				**A_     ):





        '''simple docstring'''

        kwargs.update(self.special_tokens_map     )
        return BartphoTokenizer.from_pretrained(self.tmpdirname   ,				**A_     )



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			'''This is a là test'''
        SCREAMING_SNAKE_CASE__											=			'''This is a<unk><unk> test'''
        return input_text, output_text




    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			BartphoTokenizer(A_   ,				self.monolingual_vocab_file   ,				**self.special_tokens_map     )
        SCREAMING_SNAKE_CASE__											=			'''This is a là test'''
        SCREAMING_SNAKE_CASE__											=			'''▁This ▁is ▁a ▁l à ▁t est'''.split()
        SCREAMING_SNAKE_CASE__											=			tokenizer.tokenize(A_     )
        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			tokens + [tokenizer.unk_token]
        SCREAMING_SNAKE_CASE__											=			[4, 5, 6, 3, 3, 7, 8, 3]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(A_     )   ,				A_     )
