






'''simple docstring'''



# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
      import torch
class UpperCAmelCase_       (     TensorFormatter[Mapping, '''torch.Tensor''', Mapping]					):




      """simple docstring"""




      def __init__(			self      :		Dict  ,   UpperCAmelCase      :		str=None  ,   **UpperCAmelCase      :		List[Any]		)	->	int:


       '''simple docstring'''

       super().__init__(features=UpperCAmelCase		)
       lowercase       : List[Any]						=torch_tensor_kwargs
       import torch  # noqa import torch at initialization






      def 					A__   (			self      :		Union[str, Any]  ,   UpperCAmelCase      :		Dict		)	->	List[str]:


       '''simple docstring'''

       import torch

       if isinstance(UpperCAmelCase  ,   UpperCAmelCase		) and column:
        if all(
            isinstance(UpperCAmelCase  ,   torch.Tensor		) and x.shape == column[0].shape and x.dtype == column[0].dtype
            for x in column		):
         return torch.stack(UpperCAmelCase		)
       return column






      def 					A__   (			self      :		List[str]  ,   UpperCAmelCase      :		Optional[int]		)	->	Optional[Any]:


       '''simple docstring'''

       import torch

       if isinstance(UpperCAmelCase  ,   (str, bytes, type(UpperCAmelCase		))		):
        return value
       elif isinstance(UpperCAmelCase  ,   (np.character, np.ndarray)		) and np.issubdtype(value.dtype  ,   np.character		):
        return value.tolist()

       lowercase       : Any						={}

       if isinstance(UpperCAmelCase  ,   (np.number, np.ndarray)		) and np.issubdtype(value.dtype  ,   np.integer		):
        lowercase       : Tuple						={'''dtype''': torch.intaa}
       elif isinstance(UpperCAmelCase  ,   (np.number, np.ndarray)		) and np.issubdtype(value.dtype  ,   np.floating		):
        lowercase       : int						={'''dtype''': torch.floataa}
       elif config.PIL_AVAILABLE and "PIL" in sys.modules:
        import PIL.Image

        if isinstance(UpperCAmelCase  ,   PIL.Image.Image		):
         lowercase       : List[Any]						=np.asarray(UpperCAmelCase		)
       return torch.tensor(UpperCAmelCase  ,   **{**default_dtype, **self.torch_tensor_kwargs}		)






      def 					A__   (			self      :		List[Any]  ,   UpperCAmelCase      :		Dict		)	->	Tuple:


       '''simple docstring'''

       import torch

       # support for torch, tf, jax etc.
       if hasattr(UpperCAmelCase  ,   '''__array__'''		) and not isinstance(UpperCAmelCase  ,   torch.Tensor		):
        lowercase       : List[Any]						=data_struct.__array__()
       # support for nested types like struct of list of struct
       if isinstance(UpperCAmelCase  ,   np.ndarray		):
        if data_struct.dtype == object:  # torch tensors cannot be instantied from an array of objects
         return self._consolidate([self.recursive_tensorize(UpperCAmelCase		) for substruct in data_struct]		)
       elif isinstance(UpperCAmelCase  ,   (list, tuple)		):
        return self._consolidate([self.recursive_tensorize(UpperCAmelCase		) for substruct in data_struct]		)
       return self._tensorize(UpperCAmelCase		)






      def 					A__   (			self      :		str  ,   UpperCAmelCase      :		dict		)	->	Any:


       '''simple docstring'''

       return map_nested(self._recursive_tensorize  ,   UpperCAmelCase  ,   map_list=UpperCAmelCase		)






      def 					A__   (			self      :		Union[str, Any]  ,   UpperCAmelCase      :		pa.Table		)	->	Mapping:


       '''simple docstring'''

       lowercase       : List[str]						=self.numpy_arrow_extractor().extract_row(UpperCAmelCase		)
       lowercase       : Any						=self.python_features_decoder.decode_row(UpperCAmelCase		)
       return self.recursive_tensorize(UpperCAmelCase		)






      def 					A__   (			self      :		Any  ,   UpperCAmelCase      :		pa.Table		)	->	"torch.Tensor":


       '''simple docstring'''

       lowercase       : int						=self.numpy_arrow_extractor().extract_column(UpperCAmelCase		)
       lowercase       : Tuple						=self.python_features_decoder.decode_column(UpperCAmelCase  ,   pa_table.column_names[0]		)
       lowercase       : Optional[Any]						=self.recursive_tensorize(UpperCAmelCase		)
       lowercase       : Any						=self._consolidate(UpperCAmelCase		)
       return column






      def 					A__   (			self      :		str  ,   UpperCAmelCase      :		pa.Table		)	->	Mapping:


       '''simple docstring'''

       lowercase       : Tuple						=self.numpy_arrow_extractor().extract_batch(UpperCAmelCase		)
       lowercase       : List[str]						=self.python_features_decoder.decode_batch(UpperCAmelCase		)
       lowercase       : Dict						=self.recursive_tensorize(UpperCAmelCase		)
       for column_name in batch:
        lowercase       : str						=self._consolidate(batch[column_name]		)
       return batch











'''simple docstring'''



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_xlnet import XLNetTokenizer
else:
      SCREAMING_SNAKE_CASE							=						None


SCREAMING_SNAKE_CASE							=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE							=						{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE							=						{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json',
    },
}

SCREAMING_SNAKE_CASE							=						{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

SCREAMING_SNAKE_CASE							=						'▁'

# Segments (not really needed)
SCREAMING_SNAKE_CASE							=						0
SCREAMING_SNAKE_CASE							=						1
SCREAMING_SNAKE_CASE							=						2
SCREAMING_SNAKE_CASE							=						3
SCREAMING_SNAKE_CASE							=						4
class UpperCAmelCase_       (     __A					):




      """simple docstring"""




      UpperCamelCase_             =      VOCAB_FILES_NAMES
      UpperCamelCase_             =      PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase_             =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase_             =      '''left'''
      UpperCamelCase_             =      XLNetTokenizer






      def __init__(			self      :		int  ,   UpperCAmelCase      :		Dict=None  ,   UpperCAmelCase      :		str=None  ,   UpperCAmelCase      :		str=False  ,   UpperCAmelCase      :		List[Any]=True  ,   UpperCAmelCase      :		Optional[Any]=False  ,   UpperCAmelCase      :		int="<s>"  ,   UpperCAmelCase      :		Optional[int]="</s>"  ,   UpperCAmelCase      :		str="<unk>"  ,   UpperCAmelCase      :		Optional[Any]="<sep>"  ,   UpperCAmelCase      :		Optional[int]="<pad>"  ,   UpperCAmelCase      :		Optional[Any]="<cls>"  ,   UpperCAmelCase      :		Dict="<mask>"  ,   UpperCAmelCase      :		int=["<eop>", "<eod>"]  ,   **UpperCAmelCase      :		List[Any]  ,   )	->	List[str]:


       '''simple docstring'''

       lowercase       : Dict						=AddedToken(UpperCAmelCase  ,   lstrip=UpperCAmelCase  ,   rstrip=UpperCAmelCase		) if isinstance(UpperCAmelCase  ,   UpperCAmelCase		) else mask_token

       super().__init__(
           vocab_file=UpperCAmelCase  ,   tokenizer_file=UpperCAmelCase  ,   do_lower_case=UpperCAmelCase  ,   remove_space=UpperCAmelCase  ,   keep_accents=UpperCAmelCase  ,   bos_token=UpperCAmelCase  ,   eos_token=UpperCAmelCase  ,   unk_token=UpperCAmelCase  ,   sep_token=UpperCAmelCase  ,   pad_token=UpperCAmelCase  ,   cls_token=UpperCAmelCase  ,   mask_token=UpperCAmelCase  ,   additional_special_tokens=UpperCAmelCase  ,   **UpperCAmelCase  ,   )

       lowercase       : Tuple						=3
       lowercase       : Union[str, Any]						=do_lower_case
       lowercase       : Any						=remove_space
       lowercase       : int						=keep_accents
       lowercase       : int						=vocab_file
       lowercase       : Union[str, Any]						=False if not self.vocab_file else True






      def 					A__   (			self      :		Any  ,   UpperCAmelCase      :		List[int]  ,   UpperCAmelCase      :		Optional[List[int]] = None		)	->	List[int]:


       '''simple docstring'''

       lowercase       : Union[str, Any]						=[self.sep_token_id]
       lowercase       : Optional[Any]						=[self.cls_token_id]
       if token_ids_a is None:
        return token_ids_a + sep + cls
       return token_ids_a + sep + token_ids_a + sep + cls






      def 					A__   (			self      :		str  ,   UpperCAmelCase      :		List[int]  ,   UpperCAmelCase      :		Optional[List[int]] = None		)	->	List[int]:


       '''simple docstring'''

       lowercase       : Optional[int]						=[self.sep_token_id]
       lowercase       : Union[str, Any]						=[2]

       if token_ids_a is None:
        return len(token_ids_a + sep		) * [0] + cls_segment_id
       return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1] + cls_segment_id






      def 					A__   (			self      :		str  ,   UpperCAmelCase      :		str  ,   UpperCAmelCase      :		Optional[str] = None		)	->	Tuple[str]:


       '''simple docstring'''

       if not self.can_save_slow_tokenizer:
        raise ValueError(
            '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
            '''tokenizer.'''		)

       if not os.path.isdir(UpperCAmelCase		):
        logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
        return
       lowercase       : Dict						=os.path.join(
           UpperCAmelCase  ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

       if os.path.abspath(self.vocab_file		) != os.path.abspath(UpperCAmelCase		):
        copyfile(self.vocab_file  ,   UpperCAmelCase		)

       return (out_vocab_file,)



