


# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
							import tensorflow as tf

class   A__							(						TensorFormatter[Mapping, "tf.Tensor", Mapping]     ):




						def __init__(				self    :					List[Any] ,			_a    :					List[Any]=None ,			**_a    :					List[Any]     )							->   Dict:


										"""simple docstring"""





										super().__init__(features=_a     )
										_SCREAMING_SNAKE_CASE	     =tf_tensor_kwargs
										import tensorflow as tf  # noqa: import tf at initialization




						def 							__UpperCamelCase (				self    :					Any ,			_a    :					Any     )							->   int:


										"""simple docstring"""





										import tensorflow as tf

										if isinstance(_a ,			_a     ) and column:
														if all(
														    isinstance(_a ,			tf.Tensor     ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column     ):
																		return tf.stack(_a     )
														elif all(
														    isinstance(_a ,			(tf.Tensor, tf.RaggedTensor)     ) and x.ndim == 1 and x.dtype == column[0].dtype
														    for x in column     ):
																		# only rag 1-D tensors, otherwise some dimensions become ragged even though they were consolidated
																		return tf.ragged.stack(_a     )

										return column




						def 							__UpperCamelCase (				self    :					Tuple ,			_a    :					Tuple     )							->   List[Any]:


										"""simple docstring"""





										import tensorflow as tf

										if value is None:
														return value

										_SCREAMING_SNAKE_CASE	     ={}

										if isinstance(_a ,			(np.number, np.ndarray)     ) and np.issubdtype(value.dtype ,			np.integer     ):
														_SCREAMING_SNAKE_CASE	     ={'''dtype''': tf.intaa}
										elif isinstance(_a ,			(np.number, np.ndarray)     ) and np.issubdtype(value.dtype ,			np.floating     ):
														_SCREAMING_SNAKE_CASE	     ={'''dtype''': tf.floataa}
										elif config.PIL_AVAILABLE and "PIL" in sys.modules:
														import PIL.Image

														if isinstance(_a ,			PIL.Image.Image     ):
																		_SCREAMING_SNAKE_CASE	     =np.asarray(_a     )

										return tf.convert_to_tensor(_a ,			**{**default_dtype, **self.tf_tensor_kwargs}     )




						def 							__UpperCamelCase (				self    :					Dict ,			_a    :					List[str]     )							->   Union[str, Any]:


										"""simple docstring"""





										import tensorflow as tf

										# support for torch, tf, jax etc.
										if config.TORCH_AVAILABLE and "torch" in sys.modules:
														import torch

														if isinstance(_a ,			torch.Tensor     ):
																		return self._tensorize(data_struct.detach().cpu().numpy()[()]     )
										if hasattr(_a ,			'''__array__'''     ) and not isinstance(_a ,			tf.Tensor     ):
														_SCREAMING_SNAKE_CASE	     =data_struct.__array__()
										# support for nested types like struct of list of struct
										if isinstance(_a ,			np.ndarray     ):
														if data_struct.dtype == object:  # tf tensors cannot be instantied from an array of objects
																		return self._consolidate([self.recursive_tensorize(_a     ) for substruct in data_struct]     )
										elif isinstance(_a ,			(list, tuple)     ):
														return self._consolidate([self.recursive_tensorize(_a     ) for substruct in data_struct]     )
										return self._tensorize(_a     )




						def 							__UpperCamelCase (				self    :					List[str] ,			_a    :					dict     )							->   int:


										"""simple docstring"""





										return map_nested(self._recursive_tensorize ,			_a ,			map_list=_a     )




						def 							__UpperCamelCase (				self    :					int ,			_a    :					pa.Table     )							->   Mapping:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.numpy_arrow_extractor().extract_row(_a     )
										_SCREAMING_SNAKE_CASE	     =self.python_features_decoder.decode_row(_a     )
										return self.recursive_tensorize(_a     )




						def 							__UpperCamelCase (				self    :					Any ,			_a    :					pa.Table     )							->   "tf.Tensor":


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.numpy_arrow_extractor().extract_column(_a     )
										_SCREAMING_SNAKE_CASE	     =self.python_features_decoder.decode_column(_a ,			pa_table.column_names[0]     )
										_SCREAMING_SNAKE_CASE	     =self.recursive_tensorize(_a     )
										_SCREAMING_SNAKE_CASE	     =self._consolidate(_a     )
										return column




						def 							__UpperCamelCase (				self    :					Optional[int] ,			_a    :					pa.Table     )							->   Mapping:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.numpy_arrow_extractor().extract_batch(_a     )
										_SCREAMING_SNAKE_CASE	     =self.python_features_decoder.decode_batch(_a     )
										_SCREAMING_SNAKE_CASE	     =self.recursive_tensorize(_a     )
										for column_name in batch:
														_SCREAMING_SNAKE_CASE	     =self._consolidate(batch[column_name]     )
										return batch



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import AddedToken, PreTrainedTokenizerFast
from ...utils import logging


snake_case_  :	Tuple							  =							logging.get_logger(__name__)

snake_case_  :	int							  =							{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

snake_case_  :	Dict							  =							{
    '''vocab_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/tokenizer.json''',
    },
}

class   A__							(						UpperCamelCase__     ):




						def __init__(				self    :					Optional[Any] ,			_a    :					Tuple=None ,			_a    :					Union[str, Any]=None ,			_a    :					Dict=False ,			_a    :					int=True ,			_a    :					str=False ,			_a    :					int="<s>" ,			_a    :					int="</s>" ,			_a    :					Optional[int]="<unk>" ,			_a    :					Dict="<sep>" ,			_a    :					Optional[int]="<pad>" ,			_a    :					str="<cls>" ,			_a    :					List[Any]="<mask>" ,			_a    :					Optional[Any]=["<eop>", "<eod>"] ,			**_a    :					Any ,			)							->   Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =AddedToken(_a ,			lstrip=_a ,			rstrip=_a     ) if isinstance(_a ,			_a     ) else mask_token

										super().__init__(
										    vocab_file=_a ,			tokenizer_file=_a ,			do_lower_case=_a ,			remove_space=_a ,			keep_accents=_a ,			bos_token=_a ,			eos_token=_a ,			unk_token=_a ,			sep_token=_a ,			pad_token=_a ,			cls_token=_a ,			mask_token=_a ,			additional_special_tokens=_a ,			**_a ,			)

										_SCREAMING_SNAKE_CASE	     =3
										_SCREAMING_SNAKE_CASE	     =do_lower_case
										_SCREAMING_SNAKE_CASE	     =remove_space
										_SCREAMING_SNAKE_CASE	     =keep_accents
										_SCREAMING_SNAKE_CASE	     =vocab_file
										_SCREAMING_SNAKE_CASE	     =False if not self.vocab_file else True

										try:
														import jieba
										except ModuleNotFoundError as error:
														raise error.__class__(
														    '''You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '''
														    '''See https://pypi.org/project/jieba/ for installation.'''     )
										_SCREAMING_SNAKE_CASE	     =jieba
										_SCREAMING_SNAKE_CASE	     =str.maketrans(''' \n''' ,			'''\u2582\u2583'''     )




						def 							__UpperCamelCase (				self    :					Any ,			_a    :					List[int] ,			_a    :					Optional[List[int]] = None     )							->   List[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =[self.sep_token_id]
										_SCREAMING_SNAKE_CASE	     =[self.cls_token_id]
										if token_ids_a is None:
														return token_ids_a + sep + cls
										return token_ids_a + sep + token_ids_a + sep + cls




						def 							__UpperCamelCase (				self    :					Dict ,			_a    :					List[int] ,			_a    :					Optional[List[int]] = None     )							->   List[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =[self.sep_token_id]
										_SCREAMING_SNAKE_CASE	     =[2]

										if token_ids_a is None:
														return len(token_ids_a + sep     ) * [0] + cls_segment_id
										return len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1] + cls_segment_id




						def 							__UpperCamelCase (				self    :					str ,			_a    :					str ,			_a    :					Optional[str] = None     )							->   Tuple[str]:


										"""simple docstring"""





										if not self.can_save_slow_tokenizer:
														raise ValueError(
														    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
														    '''tokenizer.'''     )

										if not os.path.isdir(_a     ):
														logger.error(f"Vocabulary path ({save_directory}) should be a directory"     )
														return
										_SCREAMING_SNAKE_CASE	     =os.path.join(
										    _a ,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )

										if os.path.abspath(self.vocab_file     ) != os.path.abspath(_a     ):
														copyfile(self.vocab_file ,			_a     )

										return (out_vocab_file,)




						def 							__UpperCamelCase (				self    :					Dict ,			_a    :					str ,			*_a    :					Optional[int] ,			**_a    :					Any     )							->   int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =[
										    ''' '''.join([x.translate(self.translator     ) for x in self.jieba.cut(_a ,			cut_all=_a     )]     )
										    for text in batch_text_or_text_pairs
										]
										return super()._batch_encode_plus(_a ,			*_a ,			**_a     )




						def 							__UpperCamelCase (				self    :					List[Any] ,			*_a    :					Union[str, Any] ,			**_a    :					Any     )							->   Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =super()._decode(*_a ,			**_a     )
										_SCREAMING_SNAKE_CASE	     =text.replace(''' ''' ,			''''''     ).replace('''\u2582''' ,			''' '''     ).replace('''\u2583''' ,			'''\n'''     )
										return text