






import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XGLMTokenizer, XGLMTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


A_		     =			get_tests_dir('fixtures/test_sentencepiece.model')




@require_sentencepiece
@require_tokenizers
class     _lowerCAmelCase(	__lowercase							,     unittest.TestCase ):





       """simple docstring"""







       a    :		List[Any]									=XGLMTokenizer
       a    :		Dict									=XGLMTokenizerFast
       a    :		List[str]									=True
       a    :		Optional[Any]									=True





       def       _a   ( self							):
         super().setUp()

         # We have a SentencePiece fixture for testing
         UpperCamelCase_: List[str]   				=    XGLMTokenizer(_lowerCamelCase  ,      keep_accents=_lowerCamelCase							)
         tokenizer.save_pretrained(self.tmpdirname							)





       def       _a   ( self							):
         UpperCamelCase_: str   				=    """<pad>"""
         UpperCamelCase_: List[str]   				=    1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCamelCase							)  ,      _lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: List[str]   				=    list(self.get_tokenizer().get_vocab().keys()							)

         self.assertEqual(vocab_keys[0]  ,      '<s>'							)
         self.assertEqual(vocab_keys[1]  ,      '<pad>'							)
         self.assertEqual(len(_lowerCamelCase							)  ,      1_0_0_8							)





       def       _a   ( self							):
         self.assertEqual(self.get_tokenizer().vocab_size  ,      1_0_0_8							)





       def       _a   ( self							):
         UpperCamelCase_: List[str]   				=    XGLMTokenizer(_lowerCamelCase  ,      keep_accents=_lowerCamelCase							)

         UpperCamelCase_: Dict   				=    tokenizer.tokenize('This is a test'							)
         self.assertListEqual(_lowerCamelCase  ,      ['▁This', '▁is', '▁a', '▁t', 'est']							)

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(_lowerCamelCase							)  ,      [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]  ,      )

         UpperCamelCase_: Union[str, Any]   				=    tokenizer.tokenize('I was born in 92000, and this is falsé.'							)
         self.assertListEqual(
             _lowerCamelCase  ,      [
                 SPIECE_UNDERLINE + 'I',
                 SPIECE_UNDERLINE + 'was',
                 SPIECE_UNDERLINE + 'b',
                 'or',
                 'n',
                 SPIECE_UNDERLINE + 'in',
                 SPIECE_UNDERLINE + '',
                 '9',
                 '2',
                 '0',
                 '0',
                 '0',
                 ',',
                 SPIECE_UNDERLINE + 'and',
                 SPIECE_UNDERLINE + 'this',
                 SPIECE_UNDERLINE + 'is',
                 SPIECE_UNDERLINE + 'f',
                 'al',
                 's',
                 'é',
                 '.',
             ]  ,      )
         UpperCamelCase_: Optional[int]   				=    tokenizer.convert_tokens_to_ids(_lowerCamelCase							)
         self.assertListEqual(
             _lowerCamelCase  ,      [
                 value + tokenizer.fairseq_offset
                 for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
             ]  ,      )

         UpperCamelCase_: Tuple   				=    tokenizer.convert_ids_to_tokens(_lowerCamelCase							)
         self.assertListEqual(
             _lowerCamelCase  ,      [
                 SPIECE_UNDERLINE + 'I',
                 SPIECE_UNDERLINE + 'was',
                 SPIECE_UNDERLINE + 'b',
                 'or',
                 'n',
                 SPIECE_UNDERLINE + 'in',
                 SPIECE_UNDERLINE + '',
                 '<unk>',
                 '2',
                 '0',
                 '0',
                 '0',
                 ',',
                 SPIECE_UNDERLINE + 'and',
                 SPIECE_UNDERLINE + 'this',
                 SPIECE_UNDERLINE + 'is',
                 SPIECE_UNDERLINE + 'f',
                 'al',
                 's',
                 '<unk>',
                 '.',
             ]  ,      )





       @cached_property
       def       _a   ( self							):
         return XGLMTokenizer.from_pretrained('facebook/xglm-564M'							)





       def       _a   ( self							):

         with tempfile.NamedTemporaryFile() as f:
           shutil.copyfile(_lowerCamelCase  ,      f.name							)
           UpperCamelCase_: Union[str, Any]   				=    XGLMTokenizer(f.name  ,      keep_accents=_lowerCamelCase							)
           UpperCamelCase_: str   				=    pickle.dumps(_lowerCamelCase							)
         pickle.loads(_lowerCamelCase							)





       def       _a   ( self							):

         if not self.test_rust_tokenizer:
           return

         UpperCamelCase_: Optional[int]   				=    self.get_tokenizer()
         UpperCamelCase_: Optional[Any]   				=    self.get_rust_tokenizer()

         UpperCamelCase_: Dict   				=    """I was born in 92000, and this is falsé."""

         UpperCamelCase_: Any   				=    tokenizer.tokenize(_lowerCamelCase							)
         UpperCamelCase_: Optional[Any]   				=    rust_tokenizer.tokenize(_lowerCamelCase							)
         self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: Tuple   				=    tokenizer.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
         UpperCamelCase_: str   				=    rust_tokenizer.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
         self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: Tuple   				=    self.get_rust_tokenizer()
         UpperCamelCase_: Optional[int]   				=    tokenizer.encode(_lowerCamelCase							)
         UpperCamelCase_: Optional[int]   				=    rust_tokenizer.encode(_lowerCamelCase							)
         self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)





       @slow
       def       _a   ( self							):
         UpperCamelCase_: str   				=    """Hello World!"""
         UpperCamelCase_: Optional[int]   				=    [2, 3_1_2_2_7, 4_4_4_7, 3_5]

         self.assertListEqual(_lowerCamelCase  ,      self.big_tokenizer.encode(_lowerCamelCase							)							)





       @slow
       def       _a   ( self							):
         UpperCamelCase_: Optional[int]   				=    (
             """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
             """ add words that should not exsist and be tokenized to unk, such as saoneuhaoesuth"""
         )
         # fmt: off
         UpperCamelCase_: Optional[int]   				=    [2, 1_0_1_8, 6_7, 1_1, 1_9_8_8, 2_6_1_7, 5_6_3_1, 2_7_8, 1_1, 3_4_0_7, 4_8, 7_1_6_3_0, 2_8_0_8_5, 4, 3_2_3_4, 1_5_7, 1_3, 6, 5, 6, 4, 3_5_2_6, 7_6_8, 1_5, 6_5_9, 5_7, 2_9_8, 3_9_8_3, 8_6_4, 1_2_9, 2_1, 6, 5, 1_3_6_7_5, 3_7_7, 6_5_2, 7_5_8_0, 1_0_3_4_1, 1_5_5, 2_8_1_7, 4_2_2, 1_6_6_6, 7, 1_6_7_4, 5_3, 1_1_3, 2_0_2_2_7_7, 1_7_8_9_2, 3_3, 6_0, 8_7, 4, 3_2_3_4, 1_5_7, 6_1, 2_6_6_7, 5_2_3_7_6, 1_9, 8_8, 2_3, 7_3_5]
         # fmt: on

         self.assertListEqual(_lowerCamelCase  ,      self.big_tokenizer.encode(_lowerCamelCase							)							)





       @slow
       def       _a   ( self							):
         UpperCamelCase_: str   				=    {
             """input_ids""": [[2, 1_0_8_8_2_5, 1_1_6_3, 1_5, 8_8_0_1_0, 4_7_3, 1_5_8_9_8, 1_5_7, 1_3_6_7_2, 1_8_5_7, 3_1_2, 8, 2_3_8_0_2_1, 1_1_6_3, 5_3, 1_3_6_7_2, 1_8_5_7, 3_1_2, 8, 5_3_2_8_3, 1_8_2_3_9_6, 8, 1_8_5_6_6, 1_6, 3_6_7_3_3, 4_1_0_1, 8, 2_3_0, 2_4_4_0_1_7, 1_2_2_5_5_3, 7, 1_5, 1_3_2_5_9_7, 4, 2_9_3, 1_2_5_1_1, 7_6_1_0, 4, 3_4_1_4, 1_3_2_5_9_7, 9, 4, 3_2_3_6_1, 3_6_2, 4, 7_3_4, 2_8_5_1_2, 3_2_5_6_9, 1_8, 4, 3_2_3_6_1, 2_6_0_9_6, 1_4_9_8_2, 7_3, 1_8_7_1_5, 2_1_4_3_3, 2_3_5_2_6_1, 1_5, 4_9_2, 1_2_4_2_7, 1_6, 5_3, 1_8_7_1_5, 2_1_4_3_3, 6_5_4_5_4, 1_5, 2_3_6_5_9, 5_6_3, 1_6, 2_7_8, 5_9_7, 2_8_4_3, 5_9_5, 7_9_3_1, 1_8_2_3_9_6, 6_4_1_8_6, 2_2, 8_8_6, 5_9_5, 1_3_2_9_8_1, 5_3, 2_5_5_4_0, 3_4_4_9, 4_3_9_8_2, 3_9_9_0_1, 5_9_5_1, 8_7_8, 3_3_0, 4, 2_7_6_9_4, 8_0_2_6_9, 3_1_2, 5_3, 6_5_1_7, 1_1_7_8_0, 6_1_1, 2_0_4_0_8, 5], [2, 6, 1_3_2_5_9_7, 6_7, 4_2_8_9_7, 3_3, 5_9_2, 8, 1_6_3_7_2_9, 2_5_5_4_0, 3_6_1, 1_3_6_9_9_7, 1_0_9_5_1_4, 1_7_3_2_3_0, 7, 5_0_1, 6_0, 1_0_2_9_1_3, 1_9_6, 5_6_3_1, 2_3_5, 6_3_2_4_3, 4_7_3, 6, 2_3_1_7_5_7, 7_4, 5_2_7_7, 7_9_0_5, 5_3, 3_0_9_5, 3_7_3_1_7, 2_2, 4_5_4, 1_8_3_8_7_4, 5], [2, 2_6_8, 3_1_2_9_8, 4_6_5_3_0, 6, 1_3_2_9_3_5, 4_3_8_3_1, 7, 5_9_7, 3_2, 2_4, 3_6_8_8, 9_8_6_5, 5]],
             """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
         }  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=_lowerCamelCase  ,      model_name='facebook/xglm-564M'  ,      padding=_lowerCamelCase  ,      )







import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment

def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__=()     ,   UpperCAmelCase__=None     ,   UpperCAmelCase__="no"     ,   UpperCAmelCase__="29500"  )	->    List[Any]:
  UpperCamelCase_: Any   				=    False
  UpperCamelCase_: List[str]   				=    False
  if any(key.startswith('KAGGLE'  ) for key in os.environ.keys()  ):
    UpperCamelCase_: List[Any]   				=    True
  elif "IPython" in sys.modules:
    UpperCamelCase_: List[Any]   				=    'google.colab' in str(sys.modules['IPython'].get_ipython()  )

  try:
    UpperCamelCase_: Optional[int]   				=    PrecisionType(mixed_precision.lower()  )
  except ValueError:
    raise ValueError(
        F'''Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.'''  )

  if (in_colab or in_kaggle) and (os.environ.get('TPU_NAME'     ,   UpperCAmelCase__  ) is not None):
    # TPU launch
    import torch_xla.distributed.xla_multiprocessing as xmp

    if len(AcceleratorState._shared_state  ) > 0:
      raise ValueError(
          'To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside '
          'your training function. Restart your notebook and make sure no cells initializes an '
          '`Accelerator`.'  )
    if num_processes is None:
      UpperCamelCase_: List[str]   				=    8

    UpperCamelCase_: str   				=    PrepareForLaunch(UpperCAmelCase__     ,   distributed_type='TPU'  )
    print(F'''Launching a training on {num_processes} TPU cores.'''  )
    xmp.spawn(UpperCAmelCase__     ,   args=UpperCAmelCase__     ,   nprocs=UpperCAmelCase__     ,   start_method='fork'  )
  elif in_colab:
    # No need for a distributed launch otherwise as it's either CPU or one GPU.
    if torch.cuda.is_available():
      print('Launching training on one GPU.'  )
    else:
      print('Launching training on one CPU.'  )
    function(*UpperCAmelCase__  )
  else:
    if num_processes is None:
      raise ValueError(
          'You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call.'  )

    if num_processes > 1:
      # Multi-GPU launch
      from torch.multiprocessing import start_processes
      from torch.multiprocessing.spawn import ProcessRaisedException

      if len(AcceleratorState._shared_state  ) > 0:
        raise ValueError(
            'To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized '
            'inside your training function. Restart your notebook and make sure no cells initializes an '
            '`Accelerator`.'  )

      if torch.cuda.is_initialized():
        raise ValueError(
            'To launch a multi-GPU training from your notebook, you need to avoid running any instruction '
            'using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA '
            'function.'  )

      # torch.distributed will expect a few environment variable to be here. We set the ones common to each
      # process here (the other ones will be set be the launcher).
      with patch_environment(
          world_size=UpperCAmelCase__     ,   master_addr='127.0.01'     ,   master_port=UpperCAmelCase__     ,   mixed_precision=UpperCAmelCase__  ):
        UpperCamelCase_: str   				=    PrepareForLaunch(UpperCAmelCase__     ,   distributed_type='MULTI_GPU'  )
        print(F'''Launching training on {num_processes} GPUs.'''  )
        try:
          start_processes(UpperCAmelCase__     ,   args=UpperCAmelCase__     ,   nprocs=UpperCAmelCase__     ,   start_method='fork'  )
        except ProcessRaisedException as e:
          if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
            raise RuntimeError(
                'CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. '
                'This likely stems from an outside import causing issues once the `notebook_launcher()` is called. '
                'Please review your imports and test them when running the `notebook_launcher()` to identify '
                'which one is problematic.'  ) from e

    else:
      # No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
      if is_mps_available():
        UpperCamelCase_: Tuple   				=    '1'
        print('Launching training on MPS.'  )
      elif torch.cuda.is_available():
        print('Launching training on one GPU.'  )
      else:
        print('Launching training on CPU.'  )
      function(*UpperCAmelCase__  )

def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__=()     ,   UpperCAmelCase__=2  )	->    Optional[int]:
  from torch.multiprocessing import start_processes

  with tempfile.NamedTemporaryFile() as tmp_file:
    # torch.distributed will expect a few environment variable to be here. We set the ones common to each
    # process here (the other ones will be set be the launcher).
    with patch_environment(
        world_size=UpperCAmelCase__     ,   master_addr='127.0.01'     ,   master_port='29500'     ,   accelerate_mixed_precision='no'     ,   accelerate_debug_rdv_file=tmp_file.name     ,   accelerate_use_cpu='yes'     ,   ):
      UpperCamelCase_: str   				=    PrepareForLaunch(UpperCAmelCase__     ,   debug=UpperCAmelCase__  )
      start_processes(UpperCAmelCase__     ,   args=UpperCAmelCase__     ,   nprocs=UpperCAmelCase__     ,   start_method='fork'  )