




'''simple docstring'''




import os
import sys
import unittest


UpperCAmelCase_												=   os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, 'utils'))

import check_dummies  # noqa: E402
from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init  # noqa: E402


# Align TRANSFORMERS_PATH in check_dummies with the current path
UpperCAmelCase_												=   os.path.join(git_repo_path, 'src', 'transformers')

UpperCAmelCase_												=   '\n{0} = None\n'

UpperCAmelCase_												=   '\nclass {0}(metaclass=DummyObject):\n    _backends = {1}\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, {1})\n'


UpperCAmelCase_												=   '\ndef {0}(*args, **kwargs):\n    requires_backends({0}, {1})\n'

class     lowerCAmelCase_ (			unittest.TestCase ):







  '''simple docstring'''







  def 				SCREAMING_SNAKE_CASE__      (    self     :						str			):






   """simple docstring"""






   UpperCAmelCase__      				=     find_backend("""    _import_structure[\"models.albert\"].append(\"AlbertTokenizerFast\")"""			)
   self.assertIsNone(snake_case_			)

   UpperCAmelCase__      				=     find_backend("""    if not is_tokenizers_available():"""			)
   self.assertEqual(snake_case_    ,		"""tokenizers"""			)

   UpperCAmelCase__      				=     find_backend("""    if not is_tensorflow_text_available():"""			)
   self.assertEqual(snake_case_    ,		"""tensorflow_text"""			)

   UpperCAmelCase__      				=     find_backend("""    if not (is_sentencepiece_available() and is_tokenizers_available()):"""			)
   self.assertEqual(snake_case_    ,		"""sentencepiece_and_tokenizers"""			)

   UpperCAmelCase__      				=     find_backend(
       """    if not (is_sentencepiece_available() and is_tensorflow_text_available()):"""			)
   self.assertEqual(snake_case_    ,		"""sentencepiece_and_tensorflow_text"""			)

   UpperCAmelCase__      				=     find_backend(
       """    if not (is_sentencepiece_available() and is_tokenizers_available() and is_vision_available()):"""			)
   self.assertEqual(snake_case_    ,		"""sentencepiece_and_tokenizers_and_vision"""			)


  def 				SCREAMING_SNAKE_CASE__      (    self     :						List[Any]			):






   """simple docstring"""






   UpperCAmelCase__      				=     read_init()
   # We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects
   self.assertIn("""torch"""    ,		snake_case_			)
   self.assertIn("""tensorflow_text"""    ,		snake_case_			)
   self.assertIn("""sentencepiece_and_tokenizers"""    ,		snake_case_			)

   # Likewise, we can't assert on the exact content of a key
   self.assertIn("""BertModel"""    ,		objects["""torch"""]			)
   self.assertIn("""TFBertModel"""    ,		objects["""tf"""]			)
   self.assertIn("""FlaxBertModel"""    ,		objects["""flax"""]			)
   self.assertIn("""BertModel"""    ,		objects["""torch"""]			)
   self.assertIn("""TFBertTokenizer"""    ,		objects["""tensorflow_text"""]			)
   self.assertIn("""convert_slow_tokenizer"""    ,		objects["""sentencepiece_and_tokenizers"""]			)


  def 				SCREAMING_SNAKE_CASE__      (    self     :						Dict			):






   """simple docstring"""






   UpperCAmelCase__      				=     create_dummy_object("""CONSTANT"""    ,		"""'torch'"""			)
   self.assertEqual(snake_case_    ,		"""\nCONSTANT = None\n"""			)

   UpperCAmelCase__      				=     create_dummy_object("""function"""    ,		"""'torch'"""			)
   self.assertEqual(
       snake_case_    ,		"""\ndef function(*args, **kwargs):\n    requires_backends(function, 'torch')\n"""			)

   UpperCAmelCase__      				=     """
class FakeClass(metaclass=DummyObject):
    _backends = 'torch'

    def __init__(self, *args, **kwargs):
        requires_backends(self, 'torch')
"""
   UpperCAmelCase__      				=     create_dummy_object("""FakeClass"""    ,		"""'torch'"""			)
   self.assertEqual(snake_case_    ,		snake_case_			)



  def 				SCREAMING_SNAKE_CASE__      (    self     :						str			):






   """simple docstring"""






   UpperCAmelCase__      				=     """# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..utils import DummyObject, requires_backends


CONSTANT = None


def function(*args, **kwargs):
    requires_backends(function, [\"torch\"])


class FakeClass(metaclass=DummyObject):
    _backends = [\"torch\"]

    def __init__(self, *args, **kwargs):
        requires_backends(self, [\"torch\"])
"""
   UpperCAmelCase__      				=     create_dummy_files({"""torch""": ["""CONSTANT""", """function""", """FakeClass"""]}			)
   self.assertEqual(dummy_files["""torch"""]    ,		snake_case_			)

"""simple docstring"""



import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase								=		logging.get_logger(__name__)

_UpperCamelCase								=		{"vocab_file": "sentencepiece.model"}

_UpperCamelCase								=		{
    "vocab_file": {
        "google/rembert": "https://huggingface.co/google/rembert/resolve/main/sentencepiece.model",
    },
}

_UpperCamelCase								=		{
    "google/rembert": 2_56,
}
class 						__UpperCAmelCase    (__A     ):







			'''simple docstring'''
			_UpperCamelCase  :						Dict					     =				VOCAB_FILES_NAMES
			_UpperCamelCase  :						List[str]					     =				PRETRAINED_VOCAB_FILES_MAP
			_UpperCamelCase  :						Any					     =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES





			def __init__(			self   ,      snake_case_   ,      snake_case_=False   ,      snake_case_=True   ,      snake_case_=True   ,      snake_case_="[CLS]"   ,      snake_case_="[SEP]"   ,      snake_case_="[UNK]"   ,      snake_case_="[SEP]"   ,      snake_case_="[PAD]"   ,      snake_case_="[CLS]"   ,      snake_case_="[MASK]"   ,      **snake_case_   ,      ):







										'''simple docstring'''


										super().__init__(
										    do_lower_case=snake_case_   ,      remove_space=snake_case_   ,      keep_accents=snake_case_   ,      bos_token=snake_case_   ,      eos_token=snake_case_   ,      unk_token=snake_case_   ,      sep_token=snake_case_   ,      pad_token=snake_case_   ,      cls_token=snake_case_   ,      mask_token=snake_case_   ,      **snake_case_   ,      )

										A__       :     List[Any]								=			do_lower_case
										A__       :     Dict								=			remove_space
										A__       :     Optional[int]								=			keep_accents
										A__       :     Tuple								=			vocab_file

										A__       :     List[str]								=			spm.SentencePieceProcessor()
										self.sp_model.Load(snake_case_							)





			@property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return len(self.sp_model							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     Optional[Any]								=			{self.convert_ids_to_tokens(snake_case_							): i for i in range(self.vocab_size							)}
										vocab.update(self.added_tokens_encoder							)
										return vocab





			def __getstate__(			self							):







										'''simple docstring'''


										A__       :     str								=			self.__dict__.copy()
										A__       :     Tuple								=			None
										return state





			def __setstate__(			self   ,      snake_case_							):







										'''simple docstring'''


										A__       :     Dict								=			d
										A__       :     int								=			spm.SentencePieceProcessor()
										self.sp_model.Load(self.vocab_file							)





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_=False							):







										'''simple docstring'''


										A__       :     Tuple								=			self.sp_model.EncodeAsPieces(snake_case_							)
										return pieces





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										return self.sp_model.PieceToId(snake_case_							)





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										return self.sp_model.IdToPiece(snake_case_							)





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										A__       :     str								=			self.sp_model.decode_pieces(snake_case_							)
										return out_string





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = None							):







										'''simple docstring'''


										A__       :     List[Any]								=			[self.sep_token_id]
										A__       :     str								=			[self.cls_token_id]
										if token_ids_a is None:
																	return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = None   ,      snake_case_ = False							):







										'''simple docstring'''


										if already_has_special_tokens:
																	if token_ids_a is not None:
																								raise ValueError(
																								    """You should not supply a second sequence if the provided sequence of """
																								    """ids is already formatted with special tokens for the model."""							)
																	return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

										if token_ids_a is not None:
																	return [1] + ([0] * len(snake_case_							)) + [1] + ([0] * len(snake_case_							)) + [1]
										return [1] + ([0] * len(snake_case_							)) + [1]





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = None							):







										'''simple docstring'''


										A__       :     int								=			[self.sep_token_id]
										A__       :     Any								=			[self.cls_token_id]

										if token_ids_a is None:
																	return len(cls + token_ids_a + sep							) * [0]
										return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = None							):







										'''simple docstring'''


										if not os.path.isdir(snake_case_							):
																	logger.error("""Vocabulary path ({}) should be a directory""".format(snake_case_							)							)
																	return
										A__       :     Any								=			os.path.join(
										    snake_case_   ,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

										if os.path.abspath(self.vocab_file							) != os.path.abspath(snake_case_							):
																	copyfile(self.vocab_file   ,      snake_case_							)

										return (out_vocab_file,)





