



"""simple docstring"""






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case   					=				logging.get_logger(__name__)
snake_case   					=				{'''vocab_file''': '''spiece.model'''}

snake_case   					=				{
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    }
}

snake_case   					=				{
    '''albert-base-v1''': 5_1_2,
    '''albert-large-v1''': 5_1_2,
    '''albert-xlarge-v1''': 5_1_2,
    '''albert-xxlarge-v1''': 5_1_2,
    '''albert-base-v2''': 5_1_2,
    '''albert-large-v2''': 5_1_2,
    '''albert-xlarge-v2''': 5_1_2,
    '''albert-xxlarge-v2''': 5_1_2,
}

snake_case   					=				'''▁'''



class     UpperCAmelCase    (	__A  ):
   A__		:	int           =  VOCAB_FILES_NAMES
   A__		:	Any           =  PRETRAINED_VOCAB_FILES_MAP
   A__		:	Dict           =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



   def __init__(       self    :	Any							,       __lowerCamelCase    :	List[str]							,       __lowerCamelCase    :	List[str]=True							,       __lowerCamelCase    :	str=True							,       __lowerCamelCase    :	int=False							,       __lowerCamelCase    :	List[Any]="[CLS]"							,       __lowerCamelCase    :	str="[SEP]"							,       __lowerCamelCase    :	Tuple="<unk>"							,       __lowerCamelCase    :	Any="[SEP]"							,       __lowerCamelCase    :	List[str]="<pad>"							,       __lowerCamelCase    :	str="[CLS]"							,       __lowerCamelCase    :	Optional[int]="[MASK]"							,       __lowerCamelCase    :	Optional[Dict[str, Any]] = None							,       **__lowerCamelCase    :	Optional[int]							,       ):



     """simple docstring"""
     _snake_case   	=      (
         AddedToken(__lowerCamelCase							,       lstrip=__lowerCamelCase							,       rstrip=__lowerCamelCase							,       normalized=__lowerCamelCase				)
         if isinstance(__lowerCamelCase							,       __lowerCamelCase				)
         else mask_token
     )

     _snake_case   	=      {} if sp_model_kwargs is None else sp_model_kwargs

     super().__init__(
         do_lower_case=__lowerCamelCase							,       remove_space=__lowerCamelCase							,       keep_accents=__lowerCamelCase							,       bos_token=__lowerCamelCase							,       eos_token=__lowerCamelCase							,       unk_token=__lowerCamelCase							,       sep_token=__lowerCamelCase							,       pad_token=__lowerCamelCase							,       cls_token=__lowerCamelCase							,       mask_token=__lowerCamelCase							,       sp_model_kwargs=self.sp_model_kwargs							,       **__lowerCamelCase							,       )

     _snake_case   	=      do_lower_case
     _snake_case   	=      remove_space
     _snake_case   	=      keep_accents
     _snake_case   	=      vocab_file

     _snake_case   	=      spm.SentencePieceProcessor(**self.sp_model_kwargs				)
     self.sp_model.Load(__lowerCamelCase				)



   @property
   def    __UpperCAmelCase    (       self    :	Optional[Any]				):



     """simple docstring"""
     return len(self.sp_model				)



   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     _snake_case   	=      {self.convert_ids_to_tokens(__lowerCamelCase				): i for i in range(self.vocab_size				)}
     vocab.update(self.added_tokens_encoder				)
     return vocab



   def __getstate__(       self    :	Optional[int]				):



     """simple docstring"""
     _snake_case   	=      self.__dict__.copy()
     _snake_case   	=      None
     return state



   def __setstate__(       self    :	Any							,       __lowerCamelCase    :	Tuple				):



     """simple docstring"""
     _snake_case   	=      d

     # for backward compatibility
     if not hasattr(self							,       '''sp_model_kwargs'''				):
       _snake_case   	=      {}

     _snake_case   	=      spm.SentencePieceProcessor(**self.sp_model_kwargs				)
     self.sp_model.Load(self.vocab_file				)



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	List[Any]				):



     """simple docstring"""

     if self.remove_space:
       _snake_case   	=      " ".join(inputs.strip().split()				)
     else:
       _snake_case   	=      inputs
     _snake_case   	=      outputs.replace('''``'''							,       '''\"'''				).replace('''\'\''''							,       '''\"'''				)

     if not self.keep_accents:
       _snake_case   	=      unicodedata.normalize('''NFKD'''							,       __lowerCamelCase				)
       _snake_case   	=      "".join([c for c in outputs if not unicodedata.combining(__lowerCamelCase				)]				)
     if self.do_lower_case:
       _snake_case   	=      outputs.lower()

     return outputs



   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	str				):



     """simple docstring"""
     _snake_case   	=      self.preprocess_text(__lowerCamelCase				)
     _snake_case   	=      self.sp_model.encode(__lowerCamelCase							,       out_type=__lowerCamelCase				)
     _snake_case   	=      []
     for piece in pieces:
       if len(__lowerCamelCase				) > 1 and piece[-1] == str(''','''				) and piece[-2].isdigit():
         _snake_case   	=      self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCamelCase							,       ''''''				)				)
         if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
           if len(cur_pieces[0]				) == 1:
             _snake_case   	=      cur_pieces[1:]
           else:
             _snake_case   	=      cur_pieces[0][1:]
         cur_pieces.append(piece[-1]				)
         new_pieces.extend(__lowerCamelCase				)
       else:
         new_pieces.append(__lowerCamelCase				)

     return new_pieces



   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	List[str]				):



     """simple docstring"""
     return self.sp_model.PieceToId(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       __lowerCamelCase    :	Optional[Any]				):



     """simple docstring"""
     return self.sp_model.IdToPiece(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	Union[str, Any]				):



     """simple docstring"""
     _snake_case   	=      []
     _snake_case   	=      ""
     _snake_case   	=      False
     for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
         if not prev_is_special:
           out_string += " "
         out_string += self.sp_model.decode(__lowerCamelCase				) + token
         _snake_case   	=      True
         _snake_case   	=      []
       else:
         current_sub_tokens.append(__lowerCamelCase				)
         _snake_case   	=      False
     out_string += self.sp_model.decode(__lowerCamelCase				)
     return out_string.strip()



   def    __UpperCAmelCase    (       self    :	List[Any]							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None				):



     """simple docstring"""
     _snake_case   	=      [self.sep_token_id]
     _snake_case   	=      [self.cls_token_id]
     if token_ids_a is None:
       return cls + token_ids_a + sep
     return cls + token_ids_a + sep + token_ids_a + sep



   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None							,       __lowerCamelCase    :	bool = False				):



     """simple docstring"""

     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=__lowerCamelCase							,       token_ids_a=__lowerCamelCase							,       already_has_special_tokens=__lowerCamelCase				)

     if token_ids_a is not None:
       return [1] + ([0] * len(__lowerCamelCase				)) + [1] + ([0] * len(__lowerCamelCase				)) + [1]
     return [1] + ([0] * len(__lowerCamelCase				)) + [1]



   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None				):



     """simple docstring"""
     _snake_case   	=      [self.sep_token_id]
     _snake_case   	=      [self.cls_token_id]

     if token_ids_a is None:
       return len(cls + token_ids_a + sep				) * [0]
     return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]





   def    __UpperCAmelCase    (       self    :	Optional[int]							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	Optional[str] = None				):



     """simple docstring"""

     if not os.path.isdir(__lowerCamelCase				):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
       return
     _snake_case   	=      os.path.join(
         __lowerCamelCase							,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

     if os.path.abspath(self.vocab_file				) != os.path.abspath(__lowerCamelCase				) and os.path.isfile(self.vocab_file				):
       copyfile(self.vocab_file							,       __lowerCamelCase				)
     elif not os.path.isfile(self.vocab_file				):
       with open(__lowerCamelCase							,       '''wb'''				) as fi:
         _snake_case   	=      self.sp_model.serialized_model_proto()
         fi.write(__lowerCamelCase				)

     return (out_vocab_file,)








# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
			import jax
			import jaxlib

a_      =     get_logger()

a_      =     None






class _UpperCamelCase				(				TensorFormatter[Mapping, 'jax.Array', Mapping] ):
			'''simple docstring'''







			def __init__(      self				:			Optional[Any]   ,   a				:			str=None   ,   a				:			List[Any]=None   ,   **a				:			Any						)				->							Optional[Any]:

				"""simple docstring"""

				super().__init__(features=a						)
				import jax
				from jaxlib.xla_client import Device

				if isinstance(a   ,   a						):
					raise ValueError(
					    F"Expected {device} to be a `str` not {type(a						)}, as `jaxlib.xla_extension.Device` "
					    "is not serializable neither with `pickle` nor with `dill`. Instead you can surround "
					    "the device with `str()` to get its string identifier that will be internally mapped "
					    "to the actual `jaxlib.xla_extension.Device`."						)
				SCREAMING_SNAKE_CASE :   List[str]					  =							device if isinstance(a   ,   a						) else str(jax.devices()[0]						)
				# using global variable since `jaxlib.xla_extension.Device` is not serializable neither
				# with `pickle` nor with `dill`, so we need to use a global variable instead
				global DEVICE_MAPPING
				if DEVICE_MAPPING is None:
					SCREAMING_SNAKE_CASE :   str					  =							self._map_devices_to_str()
				if self.device not in list(DEVICE_MAPPING.keys()						):
					logger.warning(
					    F"Device with string identifier {self.device} not listed among the available "
					    F"devices: {list(DEVICE_MAPPING.keys()						)}, so falling back to the default "
					    F"device: {str(jax.devices()[0]						)}."						)
					SCREAMING_SNAKE_CASE :   Any					  =							str(jax.devices()[0]						)
				SCREAMING_SNAKE_CASE :   Any					  =							jnp_array_kwargs
			@staticmethod
			def 				__UpperCamelCase			(      )				->							Dict[str, "jaxlib.xla_extension.Device"]:

				"""simple docstring"""

				import jax

				return {str(a						): device for device in jax.devices()}
			def 				__UpperCamelCase			(      self				:			Dict   ,   a				:			Tuple						)				->							str:

				"""simple docstring"""

				import jax
				import jax.numpy as jnp

				if isinstance(a   ,   a						) and column:
					if all(
					    isinstance(a   ,   jax.Array						) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column						):
						return jnp.stack(a   ,   axis=0						)
				return column
			def 				__UpperCamelCase			(      self				:			Dict   ,   a				:			str						)				->							str:

				"""simple docstring"""

				import jax
				import jax.numpy as jnp

				if isinstance(a   ,   (str, bytes, type(a						))						):
					return value
				elif isinstance(a   ,   (np.character, np.ndarray)						) and np.issubdtype(value.dtype   ,   np.character						):
					return value.tolist()

				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							{}

				if isinstance(a   ,   (np.number, np.ndarray)						) and np.issubdtype(value.dtype   ,   np.integer						):
					# the default int precision depends on the jax config
					# see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
					if jax.config.jax_enable_xaa:
						SCREAMING_SNAKE_CASE :   Dict					  =							{"dtype": jnp.intaa}
					else:
						SCREAMING_SNAKE_CASE :   str					  =							{"dtype": jnp.intaa}
				elif isinstance(a   ,   (np.number, np.ndarray)						) and np.issubdtype(value.dtype   ,   np.floating						):
					SCREAMING_SNAKE_CASE :   int					  =							{"dtype": jnp.floataa}
				elif config.PIL_AVAILABLE and "PIL" in sys.modules:
					import PIL.Image

					if isinstance(a   ,   PIL.Image.Image						):
						SCREAMING_SNAKE_CASE :   Dict					  =							np.asarray(a						)

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
				global DEVICE_MAPPING
				if DEVICE_MAPPING is None:
					SCREAMING_SNAKE_CASE :   Optional[Any]					  =							self._map_devices_to_str()

				with jax.default_device(DEVICE_MAPPING[self.device]						):
					# calling jnp.array on a np.ndarray does copy the data
					# see https://github.com/google/jax/issues/4486
					return jnp.array(a   ,   **{**default_dtype, **self.jnp_array_kwargs}						)
			def 				__UpperCamelCase			(      self				:			Any   ,   a				:			List[str]						)				->							Dict:

				"""simple docstring"""

				import jax

				# support for torch, tf, jax etc.
				if config.TORCH_AVAILABLE and "torch" in sys.modules:
					import torch

					if isinstance(a   ,   torch.Tensor						):
						return self._tensorize(data_struct.detach().cpu().numpy()[()]						)
				if hasattr(a   ,   "__array__"						) and not isinstance(a   ,   jax.Array						):
					SCREAMING_SNAKE_CASE :   Optional[int]					  =							data_struct.__array__()
				# support for nested types like struct of list of struct
				if isinstance(a   ,   np.ndarray						):
					if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
						return self._consolidate([self.recursive_tensorize(a						) for substruct in data_struct]						)
				elif isinstance(a   ,   (list, tuple)						):
					return self._consolidate([self.recursive_tensorize(a						) for substruct in data_struct]						)
				return self._tensorize(a						)
			def 				__UpperCamelCase			(      self				:			Optional[Any]   ,   a				:			dict						)				->							Dict:

				"""simple docstring"""

				return map_nested(self._recursive_tensorize   ,   a   ,   map_list=a						)
			def 				__UpperCamelCase			(      self				:			Dict   ,   a				:			pa.Table						)				->							Mapping:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   Optional[Any]					  =							self.numpy_arrow_extractor().extract_row(a						)
				SCREAMING_SNAKE_CASE :   List[Any]					  =							self.python_features_decoder.decode_row(a						)
				return self.recursive_tensorize(a						)
			def 				__UpperCamelCase			(      self				:			Optional[int]   ,   a				:			pa.Table						)				->							"jax.Array":

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   Optional[int]					  =							self.numpy_arrow_extractor().extract_column(a						)
				SCREAMING_SNAKE_CASE :   Optional[Any]					  =							self.python_features_decoder.decode_column(a   ,   pa_table.column_names[0]						)
				SCREAMING_SNAKE_CASE :   Tuple					  =							self.recursive_tensorize(a						)
				SCREAMING_SNAKE_CASE :   Optional[int]					  =							self._consolidate(a						)
				return column






			def 				__UpperCamelCase			(      self				:			List[Any]   ,   a				:			pa.Table						)				->							Mapping:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   str					  =							self.numpy_arrow_extractor().extract_batch(a						)
				SCREAMING_SNAKE_CASE :   str					  =							self.python_features_decoder.decode_batch(a						)
				SCREAMING_SNAKE_CASE :   List[Any]					  =							self.recursive_tensorize(a						)
				for column_name in batch:
					SCREAMING_SNAKE_CASE :   List[Any]					  =							self._consolidate(batch[column_name]						)
				return batch