


'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase       							=				logging.get_logger(__name__)

__UpperCAmelCase       							=				{'''vocab_file''': '''sentencepiece.model'''}

__UpperCAmelCase       							=				{
    '''vocab_file''': {
        '''google/rembert''': '''https://huggingface.co/google/rembert/resolve/main/sentencepiece.model''',
    },
}

__UpperCAmelCase       							=				{
    '''google/rembert''': 256,
}




class 						a__							(				a__    ):





							'''simple docstring'''






							lowercase__     :  str      		= VOCAB_FILES_NAMES
							lowercase__     :  Dict      		= PRETRAINED_VOCAB_FILES_MAP
							lowercase__     :  Optional[int]      		= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







							def __init__(						self   ,       lowerCamelCase_   ,       lowerCamelCase_=False   ,       lowerCamelCase_=True   ,       lowerCamelCase_=True   ,       lowerCamelCase_="[CLS]"   ,       lowerCamelCase_="[SEP]"   ,       lowerCamelCase_="[UNK]"   ,       lowerCamelCase_="[SEP]"   ,       lowerCamelCase_="[PAD]"   ,       lowerCamelCase_="[CLS]"   ,       lowerCamelCase_="[MASK]"   ,       **lowerCamelCase_   ,       )			->				Tuple:
														super().__init__(
														    do_lower_case=lowerCamelCase_   ,       remove_space=lowerCamelCase_   ,       keep_accents=lowerCamelCase_   ,       bos_token=lowerCamelCase_   ,       eos_token=lowerCamelCase_   ,       unk_token=lowerCamelCase_   ,       sep_token=lowerCamelCase_   ,       pad_token=lowerCamelCase_   ,       cls_token=lowerCamelCase_   ,       mask_token=lowerCamelCase_   ,       **lowerCamelCase_   ,       )

														lowerCAmelCase__			  =      do_lower_case
														lowerCAmelCase__			  =      remove_space
														lowerCAmelCase__			  =      keep_accents
														lowerCAmelCase__			  =      vocab_file

														lowerCAmelCase__			  =      spm.SentencePieceProcessor()
														self.sp_model.Load(lowerCamelCase_  )







							@property
							def     __SCREAMING_SNAKE_CASE				(						self  )			->				Any:
														return len(self.sp_model  )







							def     __SCREAMING_SNAKE_CASE				(						self  )			->				List[Any]:
														lowerCAmelCase__			  =      {self.convert_ids_to_tokens(lowerCamelCase_  ): i for i in range(self.vocab_size  )}
														vocab.update(self.added_tokens_encoder  )
														return vocab







							def __getstate__(						self  )			->				Dict:
														lowerCAmelCase__			  =      self.__dict__.copy()
														lowerCAmelCase__			  =      None
														return state







							def __setstate__(						self   ,       lowerCamelCase_  )			->				str:
														lowerCAmelCase__			  =      d
														lowerCAmelCase__			  =      spm.SentencePieceProcessor()
														self.sp_model.Load(self.vocab_file  )







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_   ,       lowerCamelCase_=False  )			->				int:
														lowerCAmelCase__			  =      self.sp_model.EncodeAsPieces(lowerCamelCase_  )
														return pieces







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_  )			->				Any:
														return self.sp_model.PieceToId(lowerCamelCase_  )







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_  )			->				Tuple:
														return self.sp_model.IdToPiece(lowerCamelCase_  )







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_  )			->				Union[str, Any]:
														lowerCAmelCase__			  =      self.sp_model.decode_pieces(lowerCamelCase_  )
														return out_string







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_   ,       lowerCamelCase_ = None  )			->				List[int]:
														lowerCAmelCase__			  =      [self.sep_token_id]
														lowerCAmelCase__			  =      [self.cls_token_id]
														if token_ids_a is None:
																					return cls + token_ids_a + sep
														return cls + token_ids_a + sep + token_ids_a + sep







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_   ,       lowerCamelCase_ = None   ,       lowerCamelCase_ = False  )			->				List[int]:

														if already_has_special_tokens:
																					if token_ids_a is not None:
																												raise ValueError(
																												    '''You should not supply a second sequence if the provided sequence of '''
																												    '''ids is already formatted with special tokens for the model.'''  )
																					return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

														if token_ids_a is not None:
																					return [1] + ([0] * len(lowerCamelCase_  )) + [1] + ([0] * len(lowerCamelCase_  )) + [1]
														return [1] + ([0] * len(lowerCamelCase_  )) + [1]







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_   ,       lowerCamelCase_ = None  )			->				List[int]:
														lowerCAmelCase__			  =      [self.sep_token_id]
														lowerCAmelCase__			  =      [self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep  ) * [0]
														return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_   ,       lowerCamelCase_ = None  )			->				Tuple[str]:
														if not os.path.isdir(lowerCamelCase_  ):
																					logger.error('''Vocabulary path ({}) should be a directory'''.format(lowerCamelCase_  )  )
																					return
														lowerCAmelCase__			  =      os.path.join(
														    lowerCamelCase_   ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )

														if os.path.abspath(self.vocab_file  ) != os.path.abspath(lowerCamelCase_  ):
																					copyfile(self.vocab_file   ,       lowerCamelCase_  )

														return (out_vocab_file,)





'''simple docstring'''






# This code is adapted from OpenAI's release
# https://github.com/openai/human-eval/blob/master/human_eval/execution.py

import contextlib
import faulthandler
import io
import multiprocessing
import os
import platform
import signal
import tempfile






def 		UpperCamelCase__      (		__SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE		)    ->   Dict:
      snake_case__       :   int	      =    multiprocessing.Manager()
      snake_case__       :   Any	      =    manager.list()

      snake_case__       :   List[str]	      =    multiprocessing.Process(target=__SCREAMING_SNAKE_CASE				,      args=(check_program, result, timeout)		)
      p.start()
      p.join(timeout=timeout + 1		)
      if p.is_alive():
            p.kill()

      if not result:
            result.append('timed out'		)

      return {
          "task_id": task_id,
          "passed": result[0] == "passed",
          "result": result[0],
          "completion_id": completion_id,
      }






def 		UpperCamelCase__      (		__SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE		)    ->   Tuple:
      with create_tempdir():
            # These system calls are needed when cleaning up tempdir.
            import os
            import shutil

            snake_case__       :   List[Any]	      =    shutil.rmtree
            snake_case__       :   Tuple	      =    os.rmdir
            snake_case__       :   Any	      =    os.chdir

            # Disable functionalities that can make destructive changes to the test.
            reliability_guard()

            # Run program.
            try:
                  snake_case__       :   Union[str, Any]	      =    {}
                  with swallow_io():
                        with time_limit(__SCREAMING_SNAKE_CASE		):
                              exec(__SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE		)
                  result.append('passed'		)
            except TimeoutException:
                  result.append('timed out'		)
            except BaseException as e:
                  result.append(f"failed: {e}"		)

            # Needed for cleaning up.
            snake_case__       :   List[Any]	      =    rmtree
            snake_case__       :   Optional[int]	      =    rmdir
            snake_case__       :   Dict	      =    chdir






@contextlib.contextmanager
def 		UpperCamelCase__      (		__SCREAMING_SNAKE_CASE		)    ->   List[str]:
      def signal_handler(__SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE		):
            raise TimeoutException('Timed out!'		)

      signal.setitimer(signal.ITIMER_REAL				,      __SCREAMING_SNAKE_CASE		)
      signal.signal(signal.SIGALRM				,      __SCREAMING_SNAKE_CASE		)
      try:
            yield
      finally:
            signal.setitimer(signal.ITIMER_REAL				,      0		)






@contextlib.contextmanager
def 		UpperCamelCase__      (		)    ->   List[str]:
      snake_case__       :   Dict	      =    WriteOnlyStringIO()
      with contextlib.redirect_stdout(__SCREAMING_SNAKE_CASE		):
            with contextlib.redirect_stderr(__SCREAMING_SNAKE_CASE		):
                  with redirect_stdin(__SCREAMING_SNAKE_CASE		):
                        yield






@contextlib.contextmanager
def 		UpperCamelCase__      (		)    ->   Optional[int]:
      with tempfile.TemporaryDirectory() as dirname:
            with chdir(__SCREAMING_SNAKE_CASE		):
                  yield dirname






class        lowercase_      (						lowerCAmelCase_							):
     pass






class        lowercase_      (						io.StringIO							):





     def 			_lowerCAmelCase      (     self     :					Optional[Any]     ,							*__lowerCamelCase     :					int     ,							**__lowerCamelCase     :					Optional[int]			):
           raise OSError





     def 			_lowerCAmelCase      (     self     :					int     ,							*__lowerCamelCase     :					List[Any]     ,							**__lowerCamelCase     :					Tuple			):
           raise OSError





     def 			_lowerCAmelCase      (     self     :					Any     ,							*__lowerCamelCase     :					int     ,							**__lowerCamelCase     :					Union[str, Any]			):
           raise OSError





     def 			_lowerCAmelCase      (     self     :					Tuple     ,							*__lowerCamelCase     :					List[str]     ,							**__lowerCamelCase     :					Optional[Any]			):
           return False






class        lowercase_      (						contextlib._RedirectStream							):  # type: ignore
     A_											=			"stdin"






@contextlib.contextmanager
def 		UpperCamelCase__      (		__SCREAMING_SNAKE_CASE		)    ->   List[str]:
      if root == ".":
            yield
            return
      snake_case__       :   str	      =    os.getcwd()
      os.chdir(__SCREAMING_SNAKE_CASE		)
      try:
            yield
      except BaseException as exc:
            raise exc
      finally:
            os.chdir(__SCREAMING_SNAKE_CASE		)






def 		UpperCamelCase__      (		__SCREAMING_SNAKE_CASE=None		)    ->   Any:

      if maximum_memory_bytes is not None:
            import resource

            resource.setrlimit(resource.RLIMIT_AS				,      (maximum_memory_bytes, maximum_memory_bytes)		)
            resource.setrlimit(resource.RLIMIT_DATA				,      (maximum_memory_bytes, maximum_memory_bytes)		)
            if not platform.uname().system == "Darwin":
                  resource.setrlimit(resource.RLIMIT_STACK				,      (maximum_memory_bytes, maximum_memory_bytes)		)

      faulthandler.disable()

      import builtins

      snake_case__       :   List[Any]	      =    None
      snake_case__       :   List[str]	      =    None

      import os

      snake_case__       :   List[Any]	      =    '1'

      snake_case__       :   Union[str, Any]	      =    None
      snake_case__       :   Tuple	      =    None
      snake_case__       :   Dict	      =    None
      snake_case__       :   Tuple	      =    None
      snake_case__       :   List[str]	      =    None
      snake_case__       :   Any	      =    None
      snake_case__       :   Optional[Any]	      =    None
      snake_case__       :   Union[str, Any]	      =    None
      snake_case__       :   int	      =    None
      snake_case__       :   Optional[int]	      =    None
      snake_case__       :   Union[str, Any]	      =    None
      snake_case__       :   List[Any]	      =    None
      snake_case__       :   Optional[int]	      =    None
      snake_case__       :   Union[str, Any]	      =    None
      snake_case__       :   Tuple	      =    None
      snake_case__       :   int	      =    None
      snake_case__       :   Optional[Any]	      =    None
      snake_case__       :   Optional[int]	      =    None
      snake_case__       :   List[str]	      =    None
      snake_case__       :   List[str]	      =    None
      snake_case__       :   List[str]	      =    None
      snake_case__       :   Dict	      =    None
      snake_case__       :   Optional[Any]	      =    None
      snake_case__       :   List[Any]	      =    None
      snake_case__       :   Tuple	      =    None
      snake_case__       :   Optional[int]	      =    None
      snake_case__       :   int	      =    None

      import shutil

      snake_case__       :   List[str]	      =    None
      snake_case__       :   List[Any]	      =    None
      snake_case__       :   List[Any]	      =    None

      import subprocess

      snake_case__       :   int	      =    None  # type: ignore

      snake_case__       :   Dict	      =    None

      import sys

      snake_case__       :   Tuple	      =    None
      snake_case__       :   Tuple	      =    None
      snake_case__       :   List[str]	      =    None
      snake_case__       :   Optional[int]	      =    None
      snake_case__       :   List[Any]	      =    None
