
"""simple docstring"""






from collections import OrderedDict
from typing import Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...feature_extraction_utils import FeatureExtractionMixin
from ...onnx import OnnxConfig
from ...onnx.utils import compute_effective_axis_dimension
from ...tokenization_utils_base import PreTrainedTokenizerBase
from ...utils import TensorType, logging


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							=     {
    'deepmind/language-perceiver': 'https://huggingface.co/deepmind/language-perceiver/resolve/main/config.json',
    # See all Perceiver models at https://huggingface.co/models?filter=perceiver
}





class 				a__					(						A__       ):
     UpperCAmelCase__         =      '''perceiver'''



     def __init__(							self :int      ,  _lowerCamelCase :Optional[int]=256      ,  _lowerCamelCase :Optional[Any]=1_280      ,  _lowerCamelCase :List[Any]=768      ,  _lowerCamelCase :Optional[Any]=1      ,  _lowerCamelCase :str=26      ,  _lowerCamelCase :str=8      ,  _lowerCamelCase :List[str]=8      ,  _lowerCamelCase :Union[str, Any]=None      ,  _lowerCamelCase :Optional[int]=None      ,  _lowerCamelCase :Any="kv"      ,  _lowerCamelCase :Union[str, Any]=1      ,  _lowerCamelCase :List[str]=1      ,  _lowerCamelCase :str="gelu"      ,  _lowerCamelCase :str=0.1      ,  _lowerCamelCase :Union[str, Any]=0.02      ,  _lowerCamelCase :Any=1E-1_2      ,  _lowerCamelCase :List[Any]=True      ,  _lowerCamelCase :Tuple=262      ,  _lowerCamelCase :Tuple=2_048      ,  _lowerCamelCase :List[Any]=56      ,  _lowerCamelCase :int=[368, 496]      ,  _lowerCamelCase :Optional[int]=16      ,  _lowerCamelCase :List[str]=1_920      ,  _lowerCamelCase :List[Any]=16      ,  _lowerCamelCase :Any=[1, 16, 224, 224]      ,  **_lowerCamelCase :Dict      ,  ):







      '''simple docstring'''



      super().__init__(**_lowerCamelCase   )

      UpperCamelCase_     :					str   	=num_latents
      UpperCamelCase_     :					Any   	=d_latents
      UpperCamelCase_     :					Optional[int]   	=d_model
      UpperCamelCase_     :					List[str]   	=num_blocks
      UpperCamelCase_     :					Optional[Any]   	=num_self_attends_per_block
      UpperCamelCase_     :					str   	=num_self_attention_heads
      UpperCamelCase_     :					List[Any]   	=num_cross_attention_heads
      UpperCamelCase_     :					List[str]   	=qk_channels
      UpperCamelCase_     :					List[Any]   	=v_channels
      UpperCamelCase_     :					Optional[Any]   	=cross_attention_shape_for_attention
      UpperCamelCase_     :					List[Any]   	=self_attention_widening_factor
      UpperCamelCase_     :					List[str]   	=cross_attention_widening_factor
      UpperCamelCase_     :					List[Any]   	=hidden_act
      UpperCamelCase_     :					List[str]   	=attention_probs_dropout_prob
      UpperCamelCase_     :					List[Any]   	=initializer_range
      UpperCamelCase_     :					List[str]   	=layer_norm_eps
      UpperCamelCase_     :					Optional[int]   	=use_query_residual
      # masked language modeling attributes
      UpperCamelCase_     :					Dict   	=vocab_size
      UpperCamelCase_     :					List[Any]   	=max_position_embeddings
      # image classification attributes
      UpperCamelCase_     :					int   	=image_size
      # flow attributes
      UpperCamelCase_     :					Union[str, Any]   	=train_size
      # multimodal autoencoding attributes
      UpperCamelCase_     :					List[str]   	=num_frames
      UpperCamelCase_     :					Optional[int]   	=audio_samples_per_frame
      UpperCamelCase_     :					Dict   	=samples_per_patch
      UpperCamelCase_     :					Tuple   	=output_shape





class 				a__					(						A__       ):



     @property
     def  lowerCamelCase_						(							self :Optional[int]   ):







      '''simple docstring'''



      if self.task == "multiple-choice":
       UpperCamelCase_     :					List[Any]   	={0: 'batch', 1: 'choice', 2: 'sequence'}
      else:
       UpperCamelCase_     :					List[Any]   	={0: 'batch', 1: 'sequence'}
      return OrderedDict(
          [
              ('inputs', dynamic_axis),
              ('attention_mask', dynamic_axis),
          ]   )



     @property
     def  lowerCamelCase_						(							self :Dict   ):







      '''simple docstring'''



      return 1E-4




     def  lowerCamelCase_						(							self :List[str]      ,  _lowerCamelCase :Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]      ,  _lowerCamelCase :int = -1      ,  _lowerCamelCase :int = -1      ,  _lowerCamelCase :int = -1      ,  _lowerCamelCase :bool = False      ,  _lowerCamelCase :Optional[TensorType] = None      ,  _lowerCamelCase :int = 3      ,  _lowerCamelCase :int = 40      ,  _lowerCamelCase :int = 40      ,  ):







      '''simple docstring'''



      if isinstance(_lowerCamelCase      ,  _lowerCamelCase   ):
       # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
       UpperCamelCase_     :					Optional[Any]   	=compute_effective_axis_dimension(
           _lowerCamelCase      ,  fixed_dimension=OnnxConfig.default_fixed_batch      ,  num_token_to_add=0   )
       # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
       UpperCamelCase_     :					str   	=preprocessor.num_special_tokens_to_add(_lowerCamelCase   )
       UpperCamelCase_     :					Union[str, Any]   	=compute_effective_axis_dimension(
           _lowerCamelCase      ,  fixed_dimension=OnnxConfig.default_fixed_sequence      ,  num_token_to_add=_lowerCamelCase   )
       # Generate dummy inputs according to compute batch and sequence
       UpperCamelCase_     :					Tuple   	=[' '.join(['a']   ) * seq_length] * batch_size
       UpperCamelCase_     :					int   	=dict(preprocessor(_lowerCamelCase      ,  return_tensors=_lowerCamelCase   )   )
       UpperCamelCase_     :					str   	=inputs.pop('input_ids'   )
       return inputs
      elif isinstance(_lowerCamelCase      ,  _lowerCamelCase   ) and preprocessor.model_input_names[0] == "pixel_values":
       # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
       UpperCamelCase_     :					List[str]   	=compute_effective_axis_dimension(_lowerCamelCase      ,  fixed_dimension=OnnxConfig.default_fixed_batch   )
       UpperCamelCase_     :					List[Any]   	=self._generate_dummy_images(_lowerCamelCase      ,  _lowerCamelCase      ,  _lowerCamelCase      ,  _lowerCamelCase   )
       UpperCamelCase_     :					Dict   	=dict(preprocessor(images=_lowerCamelCase      ,  return_tensors=_lowerCamelCase   )   )
       UpperCamelCase_     :					List[str]   	=inputs.pop('pixel_values'   )
       return inputs
      else:
       raise ValueError(
           'Unable to generate dummy inputs for the model. Please provide a tokenizer or a preprocessor.'   )


"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							=     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__SCREAMING_SNAKE_CASE							=     [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

__SCREAMING_SNAKE_CASE							=     {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
__SCREAMING_SNAKE_CASE							=     {F"""funnel-transformer/{name}""": 512 for name in _model_names}
__SCREAMING_SNAKE_CASE							=     {F"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}





class 				a__					(						A__       ):
     UpperCAmelCase__         =      VOCAB_FILES_NAMES
     UpperCAmelCase__         =      PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__         =      PRETRAINED_INIT_CONFIGURATION
     UpperCAmelCase__         =      FunnelTokenizer
     UpperCAmelCase__         =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__         =      2



     def __init__(							self :str      ,  _lowerCamelCase :int=None      ,  _lowerCamelCase :List[Any]=None      ,  _lowerCamelCase :Dict=True      ,  _lowerCamelCase :Optional[Any]="<unk>"      ,  _lowerCamelCase :Optional[Any]="<sep>"      ,  _lowerCamelCase :List[Any]="<pad>"      ,  _lowerCamelCase :List[Any]="<cls>"      ,  _lowerCamelCase :Union[str, Any]="<mask>"      ,  _lowerCamelCase :Any="<s>"      ,  _lowerCamelCase :Union[str, Any]="</s>"      ,  _lowerCamelCase :str=True      ,  _lowerCamelCase :Dict=True      ,  _lowerCamelCase :Tuple=None      ,  _lowerCamelCase :Tuple="##"      ,  **_lowerCamelCase :int      ,  ):







      '''simple docstring'''



      super().__init__(
          _lowerCamelCase      ,  tokenizer_file=_lowerCamelCase      ,  do_lower_case=_lowerCamelCase      ,  unk_token=_lowerCamelCase      ,  sep_token=_lowerCamelCase      ,  pad_token=_lowerCamelCase      ,  cls_token=_lowerCamelCase      ,  mask_token=_lowerCamelCase      ,  bos_token=_lowerCamelCase      ,  eos_token=_lowerCamelCase      ,  clean_text=_lowerCamelCase      ,  tokenize_chinese_chars=_lowerCamelCase      ,  strip_accents=_lowerCamelCase      ,  wordpieces_prefix=_lowerCamelCase      ,  **_lowerCamelCase      ,  )

      UpperCamelCase_     :					str   	=json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
      if (
          normalizer_state.get('lowercase'      ,  _lowerCamelCase   ) != do_lower_case
          or normalizer_state.get('strip_accents'      ,  _lowerCamelCase   ) != strip_accents
          or normalizer_state.get('handle_chinese_chars'      ,  _lowerCamelCase   ) != tokenize_chinese_chars
      ):
       UpperCamelCase_     :					int   	=getattr(_lowerCamelCase      ,  normalizer_state.pop('type'   )   )
       UpperCamelCase_     :					Dict   	=do_lower_case
       UpperCamelCase_     :					Tuple   	=strip_accents
       UpperCamelCase_     :					int   	=tokenize_chinese_chars
       UpperCamelCase_     :					Tuple   	=normalizer_class(**_lowerCamelCase   )

      UpperCamelCase_     :					Dict   	=do_lower_case



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[int]=None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output



     def  lowerCamelCase_						(							self :List[Any]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[int]   	=[self.sep_token_id]
      UpperCamelCase_     :					List[Any]   	=[self.cls_token_id]
      if token_ids_a is None:
       return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0]
      return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]




     def  lowerCamelCase_						(							self :Optional[Any]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[str] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=self._tokenizer.model.save(_lowerCamelCase      ,  name=_lowerCamelCase   )
      return tuple(_lowerCamelCase   )
