import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin




class 					_UpperCAmelCase			(     lowerCAmelCase,			unittest.TestCase  ):



 '''simple docstring'''




 __A			       =					XLMTokenizer
 __A			       =					False







 def 		__UpperCAmelCase				(		self				:   int)						->	Any:


   """simple docstring"""







   super().setUp()

   # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
   _UpperCamelCase           =    [
       "l",
       "o",
       "w",
       "e",
       "r",
       "s",
       "t",
       "i",
       "d",
       "n",
       "w</w>",
       "r</w>",
       "t</w>",
       "lo",
       "low",
       "er</w>",
       "low</w>",
       "lowest</w>",
       "newer</w>",
       "wider</w>",
       "<unk>",
   ]
   _UpperCamelCase           =    dict(zip(lowercase_							, range(len(lowercase_))))
   _UpperCamelCase           =    ["l o 123", "lo w 1456", "e r</w> 1789", ""]

   _UpperCamelCase           =    os.path.join(self.tmpdirname							, VOCAB_FILES_NAMES["vocab_file"])
   _UpperCamelCase           =    os.path.join(self.tmpdirname							, VOCAB_FILES_NAMES["merges_file"])
   with open(self.vocab_file							, "w") as fp:
     fp.write(json.dumps(lowercase_))
   with open(self.merges_file							, "w") as fp:
     fp.write("\n".join(lowercase_))







 def 		__UpperCAmelCase				(		self				:   int							, lowercase_				:   str)						->	List[str]:


   """simple docstring"""







   _UpperCamelCase           =    "lower newer"
   _UpperCamelCase           =    "lower newer"
   return input_text, output_text







 def 		__UpperCAmelCase				(		self				:   str)						->	Union[str, Any]:


   """simple docstring"""







   _UpperCamelCase           =    XLMTokenizer(self.vocab_file							, self.merges_file)

   _UpperCamelCase           =    "lower"
   _UpperCamelCase           =    ["low", "er</w>"]
   _UpperCamelCase           =    tokenizer.tokenize(lowercase_)
   self.assertListEqual(lowercase_							, lowercase_)

   _UpperCamelCase           =    tokens + ["<unk>"]
   _UpperCamelCase           =    [14, 15, 20]
   self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_)							, lowercase_)







 @slow
 def 		__UpperCAmelCase				(		self				:   Any)						->	Any:


   """simple docstring"""







   _UpperCamelCase           =    XLMTokenizer.from_pretrained("xlm-mlm-en-2048")

   _UpperCamelCase           =    tokenizer.encode("sequence builders"							, add_special_tokens=lowercase_)
   _UpperCamelCase           =    tokenizer.encode("multi-sequence build"							, add_special_tokens=lowercase_)

   _UpperCamelCase           =    tokenizer.build_inputs_with_special_tokens(lowercase_)
   _UpperCamelCase           =    tokenizer.build_inputs_with_special_tokens(lowercase_							, lowercase_)

   assert encoded_sentence == [0] + text + [1]
   assert encoded_pair == [0] + text + [1] + text_a + [1]





# This code is adapted from OpenAI's release
# https://github.com/openai/human-eval/blob/master/human_eval/execution.py

import contextlib
import faulthandler
import io
import multiprocessing
import os
import platform
import signal
import tempfile


def 				lowerCAmelCase__       (			a__    ,     a__    ,     a__    ,     a__ )      ->str:




  '''simple docstring'''

  _UpperCamelCase           =    multiprocessing.Manager()
  _UpperCamelCase           =    manager.list()

  _UpperCamelCase           =    multiprocessing.Process(target=a__    ,     args=(check_program, result, timeout) )
  p.start()
  p.join(timeout=timeout + 1 )
  if p.is_alive():
    p.kill()

  if not result:
    result.append("timed out" )

  return {
      "task_id": task_id,
      "passed": result[0] == "passed",
      "result": result[0],
      "completion_id": completion_id,
  }


def 				lowerCAmelCase__       (			a__    ,     a__    ,     a__ )      ->Union[str, Any]:




  '''simple docstring'''

  with create_tempdir():
    # These system calls are needed when cleaning up tempdir.
    import os
    import shutil

    _UpperCamelCase           =    shutil.rmtree
    _UpperCamelCase           =    os.rmdir
    _UpperCamelCase           =    os.chdir

    # Disable functionalities that can make destructive changes to the test.
    reliability_guard()

    # Run program.
    try:
      _UpperCamelCase           =    {}
      with swallow_io():
        with time_limit(a__ ):
          exec(a__    ,     a__ )
      result.append("passed" )
    except TimeoutException:
      result.append("timed out" )
    except BaseException as e:
      result.append(f'failed: {e}' )

    # Needed for cleaning up.
    _UpperCamelCase           =    rmtree
    _UpperCamelCase           =    rmdir
    _UpperCamelCase           =    chdir


@contextlib.contextmanager
def 				lowerCAmelCase__       (			a__ )      ->Optional[Any]:




  '''simple docstring'''

  def signal_handler(a__    ,     a__ ):
    raise TimeoutException("Timed out!" )

  signal.setitimer(signal.ITIMER_REAL    ,     a__ )
  signal.signal(signal.SIGALRM    ,     a__ )
  try:
    yield
  finally:
    signal.setitimer(signal.ITIMER_REAL    ,     0 )


@contextlib.contextmanager
def 				lowerCAmelCase__       (			)      ->Optional[int]:




  '''simple docstring'''

  _UpperCamelCase           =    WriteOnlyStringIO()
  with contextlib.redirect_stdout(a__ ):
    with contextlib.redirect_stderr(a__ ):
      with redirect_stdin(a__ ):
        yield


@contextlib.contextmanager
def 				lowerCAmelCase__       (			)      ->Optional[int]:




  '''simple docstring'''

  with tempfile.TemporaryDirectory() as dirname:
    with chdir(a__ ):
      yield dirname




class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''




 pass




class 					_UpperCAmelCase			(     io.StringIO  ):



 '''simple docstring'''




 def 		__UpperCAmelCase				(		self				:   List[str]							, *lowercase_				:   str							, **lowercase_				:   List[str])						->	Optional[Any]:


   """simple docstring"""







   raise OSError







 def 		__UpperCAmelCase				(		self				:   Any							, *lowercase_				:   List[Any]							, **lowercase_				:   Optional[int])						->	Optional[int]:


   """simple docstring"""







   raise OSError







 def 		__UpperCAmelCase				(		self				:   int							, *lowercase_				:   int							, **lowercase_				:   str)						->	int:


   """simple docstring"""







   raise OSError







 def 		__UpperCAmelCase				(		self				:   List[Any]							, *lowercase_				:   Tuple							, **lowercase_				:   Tuple)						->	Tuple:


   """simple docstring"""







   return False




class 					_UpperCAmelCase			(     contextlib._RedirectStream  ):  # type: ignore



 '''simple docstring'''




 __A			       =					'''stdin'''


@contextlib.contextmanager
def 				lowerCAmelCase__       (			a__ )      ->str:




  '''simple docstring'''

  if root == ".":
    yield
    return
  _UpperCamelCase           =    os.getcwd()
  os.chdir(a__ )
  try:
    yield
  except BaseException as exc:
    raise exc
  finally:
    os.chdir(a__ )







def 				lowerCAmelCase__       (			a__=None )      ->Optional[int]:




  '''simple docstring'''

  if maximum_memory_bytes is not None:
    import resource

    resource.setrlimit(resource.RLIMIT_AS    ,     (maximum_memory_bytes, maximum_memory_bytes) )
    resource.setrlimit(resource.RLIMIT_DATA    ,     (maximum_memory_bytes, maximum_memory_bytes) )
    if not platform.uname().system == "Darwin":
      resource.setrlimit(resource.RLIMIT_STACK    ,     (maximum_memory_bytes, maximum_memory_bytes) )

  faulthandler.disable()

  import builtins

  _UpperCamelCase           =    None
  _UpperCamelCase           =    None

  import os

  _UpperCamelCase           =    "1"

  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None

  import shutil

  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None

  import subprocess

  _UpperCamelCase           =    None  # type: ignore

  _UpperCamelCase           =    None

  import sys

  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None
  _UpperCamelCase           =    None




