



"""simple docstring"""
from typing import Dict, List, Optional, Union

import numpy as np

from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from .utils import PaddingStrategy, TensorType, is_tf_tensor, is_torch_tensor, logging, to_numpy


UpperCAmelCase						:		Optional[Any]			= logging.get_logger(__name__)


class    SCREAMING_SNAKE_CASE__ (							__UpperCAmelCase   ):

    def __init__(				self			:				Optional[Any] ,  lowerCAmelCase_			:				int ,  lowerCAmelCase_			:				int ,  lowerCAmelCase_			:				float ,  **lowerCAmelCase_			:				Optional[int]):







      """simple docstring"""







      lowercase_								= feature_size
      lowercase_								= sampling_rate
      lowercase_								= padding_value

      lowercase_								= kwargs.pop("""padding_side""" ,  """right""")
      lowercase_								= kwargs.pop("""return_attention_mask""" ,  lowerCAmelCase_)

      super().__init__(**lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				int ,  lowerCAmelCase_			:				Union[
            BatchFeature,
            List[BatchFeature],
            Dict[str, BatchFeature],
            Dict[str, List[BatchFeature]],
            List[Dict[str, BatchFeature]],
        ] ,  lowerCAmelCase_			:				Union[bool, str, PaddingStrategy] = True ,  lowerCAmelCase_			:				Optional[int] = None ,  lowerCAmelCase_			:				bool = False ,  lowerCAmelCase_			:				Optional[int] = None ,  lowerCAmelCase_			:				Optional[bool] = None ,  lowerCAmelCase_			:				Optional[Union[str, TensorType]] = None ,  ):







      """simple docstring"""







      if isinstance(lowerCAmelCase_ ,  (list, tuple)) and isinstance(processed_features[0] ,  (dict, BatchFeature)):
        lowercase_								= {
            key: [example[key] for example in processed_features] for key in processed_features[0].keys()
        }

      # The model's main input name, usually `input_values`, has be passed for padding
      if self.model_input_names[0] not in processed_features:
        raise ValueError(
            """You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`"""
            F''' to this method that includes {self.model_input_names[0]}, but you provided'''
            F''' {list(processed_features.keys())}''')

      lowercase_								= processed_features[self.model_input_names[0]]
      lowercase_								= (
          return_attention_mask if return_attention_mask is not None else self.return_attention_mask
      )

      if len(lowerCAmelCase_) == 0:
        if return_attention_mask:
          lowercase_								= []
        return processed_features

      # If we have PyTorch/TF tensors or lists as inputs, we cast them as Numpy arrays
      # and rebuild them afterwards if no return_tensors is specified
      # Note that we lose the specific device the tensor may be on for PyTorch

      lowercase_								= required_input[0]
      if isinstance(lowerCAmelCase_ ,  (list, tuple)):
        # first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
        lowercase_								= 0
        while len(required_input[index]) == 0:
          index += 1
        if index < len(lowerCAmelCase_):
          lowercase_								= required_input[index][0]

      if return_tensors is None:
        if is_tf_tensor(lowerCAmelCase_):
          lowercase_								= """tf"""
        elif is_torch_tensor(lowerCAmelCase_):
          lowercase_								= """pt"""
        elif isinstance(lowerCAmelCase_ ,  (int, float, list, tuple, np.ndarray)):
          lowercase_								= """np"""
        else:
          raise ValueError(
              F'''type of {first_element} unknown: {type(lowerCAmelCase_)}. '''
              """Should be one of a python, numpy, pytorch or tensorflow object.""")

      for key, value in processed_features.items():
        if isinstance(value[0] ,  (int, float)):
          lowercase_								= to_numpy(lowerCAmelCase_)
        else:
          lowercase_								= [to_numpy(lowerCAmelCase_) for v in value]

        # Convert padding_strategy in PaddingStrategy
      lowercase_								= self._get_padding_strategies(padding=lowerCAmelCase_ ,  max_length=lowerCAmelCase_)

      lowercase_								= processed_features[self.model_input_names[0]]

      lowercase_								= len(lowerCAmelCase_)
      if not all(len(lowerCAmelCase_) == batch_size for v in processed_features.values()):
        raise ValueError("""Some items in the output dictionary have a different batch size than others.""")

      lowercase_								= []
      for i in range(lowerCAmelCase_):
        lowercase_								= {k: v[i] for k, v in processed_features.items()}
        # truncation
        lowercase_								= self._truncate(
            lowerCAmelCase_ ,  max_length=lowerCAmelCase_ ,  pad_to_multiple_of=lowerCAmelCase_ ,  truncation=lowerCAmelCase_ ,  )
        truncated_inputs.append(lowerCAmelCase_)

      if padding_strategy == PaddingStrategy.LONGEST:
        # make sure that `max_length` cannot be longer than the longest truncated length
        lowercase_								= max(len(input_slice[self.model_input_names[0]]) for input_slice in truncated_inputs)
        lowercase_								= PaddingStrategy.MAX_LENGTH

      lowercase_								= {}
      for i in range(lowerCAmelCase_):
        # padding
        lowercase_								= self._pad(
            truncated_inputs[i] ,  max_length=lowerCAmelCase_ ,  padding_strategy=lowerCAmelCase_ ,  pad_to_multiple_of=lowerCAmelCase_ ,  return_attention_mask=lowerCAmelCase_ ,  )

        for key, value in outputs.items():
          if key not in batch_outputs:
            lowercase_								= []
          if value.dtype is np.dtype(np.floataa):
            lowercase_								= value.astype(np.floataa)
          batch_outputs[key].append(lowerCAmelCase_)

      return BatchFeature(lowerCAmelCase_ ,  tensor_type=lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				Dict ,  lowerCAmelCase_			:				Union[Dict[str, np.ndarray], BatchFeature] ,  lowerCAmelCase_			:				Optional[int] = None ,  lowerCAmelCase_			:				PaddingStrategy = PaddingStrategy.DO_NOT_PAD ,  lowerCAmelCase_			:				Optional[int] = None ,  lowerCAmelCase_			:				Optional[bool] = None ,  ):







      """simple docstring"""







      lowercase_								= processed_features[self.model_input_names[0]]

      if padding_strategy == PaddingStrategy.LONGEST:
        lowercase_								= len(lowerCAmelCase_)

      if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
        lowercase_								= ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

      lowercase_								= padding_strategy != PaddingStrategy.DO_NOT_PAD and len(lowerCAmelCase_) < max_length

      if return_attention_mask and "attention_mask" not in processed_features:
        lowercase_								= np.ones(len(lowerCAmelCase_) ,  dtype=np.intaa)

      if needs_to_be_padded:
        lowercase_								= max_length - len(lowerCAmelCase_)
        if self.padding_side == "right":
          if return_attention_mask:
            lowercase_								= np.pad(
                processed_features["""attention_mask"""] ,  (0, difference))
          lowercase_								= ((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference)
          lowercase_								= np.pad(
              lowerCAmelCase_ ,  lowerCAmelCase_ ,  """constant""" ,  constant_values=self.padding_value)
        elif self.padding_side == "left":
          if return_attention_mask:
            lowercase_								= np.pad(
                processed_features["""attention_mask"""] ,  (difference, 0))
          lowercase_								= ((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0)
          lowercase_								= np.pad(
              lowerCAmelCase_ ,  lowerCAmelCase_ ,  """constant""" ,  constant_values=self.padding_value)
        else:
          raise ValueError("""Invalid padding strategy:""" + str(self.padding_side))

      return processed_features

    def 					_UpperCAmelCase	(				self			:				int ,  lowerCAmelCase_			:				Union[Dict[str, np.ndarray], BatchFeature] ,  lowerCAmelCase_			:				Optional[int] = None ,  lowerCAmelCase_			:				Optional[int] = None ,  lowerCAmelCase_			:				Optional[bool] = None ,  ):







      """simple docstring"""







      if not truncation:
        return processed_features
      elif truncation and max_length is None:
        raise ValueError("""When setting ``truncation=True``, make sure that ``max_length`` is defined.""")

      lowercase_								= processed_features[self.model_input_names[0]]

      # find `max_length` that fits `pad_to_multiple_of`
      if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
        lowercase_								= ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

      lowercase_								= len(lowerCAmelCase_) > max_length

      if needs_to_be_truncated:
        lowercase_								= processed_features[self.model_input_names[0]][:max_length]
        if "attention_mask" in processed_features:
          lowercase_								= processed_features["""attention_mask"""][:max_length]

      return processed_features




    def 					_UpperCAmelCase	(				self			:				Union[str, Any] ,  lowerCAmelCase_			:				Optional[Any]=False ,  lowerCAmelCase_			:				List[Any]=None):







      """simple docstring"""







      if padding is not False:
        if padding is True:
          lowercase_								= PaddingStrategy.LONGEST  # Default to pad to the longest sequence in the batch
        elif not isinstance(lowerCAmelCase_ ,  lowerCAmelCase_):
          lowercase_								= PaddingStrategy(lowerCAmelCase_)
        elif isinstance(lowerCAmelCase_ ,  lowerCAmelCase_):
          lowercase_								= padding
      else:
        lowercase_								= PaddingStrategy.DO_NOT_PAD

      # Set max length if needed
      if max_length is None:
        if padding_strategy == PaddingStrategy.MAX_LENGTH:
          raise ValueError(
              F'''When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined''')

        # Test if we have a padding value
      if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):
        raise ValueError(
            """Asking to pad but the feature_extractor does not have a padding value. Please select a value to use"""
            """ as `padding_value`. For example: `feature_extractor.padding_value = 0.0`.""")

      return padding_strategy






"""simple docstring"""
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase						:		List[Any]			= logging.get_logger(__name__)


UpperCAmelCase						:		List[Any]			= "▁"

UpperCAmelCase						:		Union[str, Any]			= {"vocab_file": "spiece.model"}

UpperCAmelCase						:		List[Any]			= {
    "vocab_file": {
        "google/reformer-crime-and-punishment": (
            "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model"
        )
    }
}

UpperCAmelCase						:		Optional[Any]			= {
    "google/reformer-crime-and-punishment": 52_4288,
}


class    SCREAMING_SNAKE_CASE__ (							__UpperCAmelCase   ):
    lowercase__      =   VOCAB_FILES_NAMES
    lowercase__      =   PRETRAINED_VOCAB_FILES_MAP
    lowercase__      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__      =   ["input_ids", "attention_mask"]

    def __init__(				self			:				int ,  lowerCAmelCase_			:				Any ,  lowerCAmelCase_			:				Dict="</s>" ,  lowerCAmelCase_			:				Dict="<unk>" ,  lowerCAmelCase_			:				Dict=[] ,  lowerCAmelCase_			:				Optional[Dict[str, Any]] = None ,  **lowerCAmelCase_			:				List[str] ,  ):







      """simple docstring"""







      lowercase_								= {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          eos_token=lowerCAmelCase_ ,  unk_token=lowerCAmelCase_ ,  additional_special_tokens=lowerCAmelCase_ ,  sp_model_kwargs=self.sp_model_kwargs ,  **lowerCAmelCase_ ,  )

      lowercase_								= vocab_file
      lowercase_								= spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(lowerCAmelCase_)

    @property
    def 					_UpperCAmelCase	(				self			:				List[Any]):







      """simple docstring"""







      return self.sp_model.get_piece_size()

    def 					_UpperCAmelCase	(				self			:				Dict):







      """simple docstring"""







      lowercase_								= {self.convert_ids_to_tokens(lowerCAmelCase_): i for i in range(self.vocab_size)}
      vocab.update(self.added_tokens_encoder)
      return vocab

    def __getstate__(				self			:				Union[str, Any]):







      """simple docstring"""







      lowercase_								= self.__dict__.copy()
      lowercase_								= None
      return state

    def __setstate__(				self			:				Optional[Any] ,  lowerCAmelCase_			:				int):







      """simple docstring"""







      lowercase_								= d

      # for backward compatibility
      if not hasattr(self ,  """sp_model_kwargs"""):
        lowercase_								= {}

      lowercase_								= spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(self.vocab_file)

    def 					_UpperCAmelCase	(				self			:				Union[str, Any] ,  lowerCAmelCase_			:				str):







      """simple docstring"""







      return self.sp_model.encode(lowerCAmelCase_ ,  out_type=lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				Tuple ,  lowerCAmelCase_			:				List[str]):







      """simple docstring"""







      return self.sp_model.piece_to_id(lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				str ,  lowerCAmelCase_			:				Any):







      """simple docstring"""







      if index < self.sp_model.get_piece_size():
        lowercase_								= self.sp_model.IdToPiece(lowerCAmelCase_)
      return token

    def 					_UpperCAmelCase	(				self			:				int ,  lowerCAmelCase_			:				Any):







      """simple docstring"""







      lowercase_								= []
      lowercase_								= """"""
      for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
          out_string += self.sp_model.decode(lowerCAmelCase_) + token
          lowercase_								= []
        else:
          current_sub_tokens.append(lowerCAmelCase_)
      out_string += self.sp_model.decode(lowerCAmelCase_)
      return out_string.strip()




    def 					_UpperCAmelCase	(				self			:				Optional[int] ,  lowerCAmelCase_			:				str ,  lowerCAmelCase_			:				Optional[str] = None):







      """simple docstring"""







      if not os.path.isdir(lowerCAmelCase_):
        logger.error(F'''Vocabulary path ({save_directory}) should be a directory''')
        return
      lowercase_								= os.path.join(
          lowerCAmelCase_ ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])

      if os.path.abspath(self.vocab_file) != os.path.abspath(lowerCAmelCase_) and os.path.isfile(self.vocab_file):
        copyfile(self.vocab_file ,  lowerCAmelCase_)
      elif not os.path.isfile(self.vocab_file):
        with open(lowerCAmelCase_ ,  """wb""") as fi:
          lowercase_								= self.sp_model.serialized_model_proto()
          fi.write(lowerCAmelCase_)

      return (out_vocab_file,)

