
"""simple docstring"""






import numpy as np

from transformers import BatchFeature
from transformers.testing_utils import require_tf, require_torch

from .test_feature_extraction_common import FeatureExtractionSavingTestMixin





class 				a__					(						A__       ):
     # to overwrite at feature extractactor specific tests
     UpperCAmelCase__         =      None
     UpperCAmelCase__         =      None



     @property
     def  lowerCamelCase_						(							self :List[str]   ):







      '''simple docstring'''



      return self.feat_extract_tester.prepare_feat_extract_dict()



     def  lowerCamelCase_						(							self :Any   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=self.feature_extraction_class(**self.feat_extract_dict   )
      self.assertTrue(hasattr(_lowerCamelCase      ,  'feature_size'   )   )
      self.assertTrue(hasattr(_lowerCamelCase      ,  'sampling_rate'   )   )
      self.assertTrue(hasattr(_lowerCamelCase      ,  'padding_value'   )   )



     def  lowerCamelCase_						(							self :str   ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=self.feat_extract_tester.prepare_inputs_for_common()
      UpperCamelCase_     :					int   	=self.feature_extraction_class(**self.feat_extract_dict   )
      UpperCamelCase_     :					int   	=feat_extract.model_input_names[0]

      UpperCamelCase_     :					int   	=BatchFeature({input_name: speech_inputs}   )

      self.assertTrue(all(len(_lowerCamelCase   ) == len(_lowerCamelCase   ) for x, y in zip(_lowerCamelCase      ,  processed_features[input_name]   )   )   )

      UpperCamelCase_     :					str   	=self.feat_extract_tester.prepare_inputs_for_common(equal_length=_lowerCamelCase   )
      UpperCamelCase_     :					List[Any]   	=BatchFeature({input_name: speech_inputs}      ,  tensor_type='np'   )

      UpperCamelCase_     :					int   	=processed_features[input_name]

      if len(batch_features_input.shape   ) < 3:
       UpperCamelCase_     :					Optional[Any]   	=batch_features_input[:, :, None]

      self.assertTrue(
          batch_features_input.shape
          == (self.feat_extract_tester.batch_size, len(speech_inputs[0]   ), self.feat_extract_tester.feature_size)   )



     @require_torch
     def  lowerCamelCase_						(							self :List[str]   ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=self.feat_extract_tester.prepare_inputs_for_common(equal_length=_lowerCamelCase   )
      UpperCamelCase_     :					List[Any]   	=self.feature_extraction_class(**self.feat_extract_dict   )
      UpperCamelCase_     :					Dict   	=feat_extract.model_input_names[0]

      UpperCamelCase_     :					Optional[int]   	=BatchFeature({input_name: speech_inputs}      ,  tensor_type='pt'   )

      UpperCamelCase_     :					Any   	=processed_features[input_name]

      if len(batch_features_input.shape   ) < 3:
       UpperCamelCase_     :					Any   	=batch_features_input[:, :, None]

      self.assertTrue(
          batch_features_input.shape
          == (self.feat_extract_tester.batch_size, len(speech_inputs[0]   ), self.feat_extract_tester.feature_size)   )



     @require_tf
     def  lowerCamelCase_						(							self :Tuple   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=self.feat_extract_tester.prepare_inputs_for_common(equal_length=_lowerCamelCase   )
      UpperCamelCase_     :					Optional[Any]   	=self.feature_extraction_class(**self.feat_extract_dict   )
      UpperCamelCase_     :					Tuple   	=feat_extract.model_input_names[0]

      UpperCamelCase_     :					Union[str, Any]   	=BatchFeature({input_name: speech_inputs}      ,  tensor_type='tf'   )

      UpperCamelCase_     :					Dict   	=processed_features[input_name]

      if len(batch_features_input.shape   ) < 3:
       UpperCamelCase_     :					Optional[int]   	=batch_features_input[:, :, None]

      self.assertTrue(
          batch_features_input.shape
          == (self.feat_extract_tester.batch_size, len(speech_inputs[0]   ), self.feat_extract_tester.feature_size)   )



     def  lowerCamelCase_						(							self :Union[str, Any]      ,  _lowerCamelCase :List[Any]=False   ):







      '''simple docstring'''



      def _inputs_have_equal_length(_lowerCamelCase :List[Any]   ):
       UpperCamelCase_     :					List[str]   	=len(input[0]   )
       for input_slice in input[1:]:
        if len(_lowerCamelCase   ) != length:
         return False
       return True

      def _inputs_are_equal(_lowerCamelCase :str      ,  _lowerCamelCase :str   ):
       if len(_lowerCamelCase   ) != len(_lowerCamelCase   ):
        return False

       for input_slice_a, input_slice_a in zip(_lowerCamelCase      ,  _lowerCamelCase   ):
        if not np.allclose(np.asarray(_lowerCamelCase   )      ,  np.asarray(_lowerCamelCase   )      ,  atol=1E-3   ):
         return False
       return True

      UpperCamelCase_     :					int   	=self.feature_extraction_class(**self.feat_extract_dict   )
      UpperCamelCase_     :					Optional[Any]   	=self.feat_extract_tester.prepare_inputs_for_common(numpify=_lowerCamelCase   )
      UpperCamelCase_     :					Dict   	=feat_extract.model_input_names[0]

      UpperCamelCase_     :					Dict   	=BatchFeature({input_name: speech_inputs}   )

      UpperCamelCase_     :					Dict   	=self.feat_extract_tester.seq_length_diff
      UpperCamelCase_     :					List[str]   	=self.feat_extract_tester.max_seq_length + pad_diff
      UpperCamelCase_     :					str   	=self.feat_extract_tester.min_seq_length
      UpperCamelCase_     :					Tuple   	=self.feat_extract_tester.batch_size
      UpperCamelCase_     :					Dict   	=self.feat_extract_tester.feature_size

      # test padding for List[int] + numpy
      UpperCamelCase_     :					List[str]   	=feat_extract.pad(_lowerCamelCase      ,  padding=_lowerCamelCase   )
      UpperCamelCase_     :					str   	=input_a[input_name]

      UpperCamelCase_     :					Dict   	=feat_extract.pad(_lowerCamelCase      ,  padding='longest'   )
      UpperCamelCase_     :					Tuple   	=input_a[input_name]

      UpperCamelCase_     :					str   	=feat_extract.pad(_lowerCamelCase      ,  padding='max_length'      ,  max_length=len(speech_inputs[-1]   )   )
      UpperCamelCase_     :					int   	=input_a[input_name]

      UpperCamelCase_     :					Optional[int]   	=feat_extract.pad(_lowerCamelCase      ,  padding='longest'      ,  return_tensors='np'   )
      UpperCamelCase_     :					Optional[Any]   	=input_a[input_name]

      # max_length parameter has to be provided when setting `padding="max_length"`
      with self.assertRaises(_lowerCamelCase   ):
       feat_extract.pad(_lowerCamelCase      ,  padding='max_length'   )[input_name]

      UpperCamelCase_     :					Union[str, Any]   	=feat_extract.pad(
          _lowerCamelCase      ,  padding='max_length'      ,  max_length=_lowerCamelCase      ,  return_tensors='np'   )
      UpperCamelCase_     :					Dict   	=input_a[input_name]

      self.assertFalse(_inputs_have_equal_length(_lowerCamelCase   )   )
      self.assertTrue(_inputs_have_equal_length(_lowerCamelCase   )   )
      self.assertTrue(_inputs_have_equal_length(_lowerCamelCase   )   )
      self.assertTrue(_inputs_are_equal(_lowerCamelCase      ,  _lowerCamelCase   )   )
      self.assertTrue(len(input_a[0]   ) == pad_min_length   )
      self.assertTrue(len(input_a[1]   ) == pad_min_length + pad_diff   )
      self.assertTrue(input_a.shape[:2] == (batch_size, len(input_a[0]   ))   )
      self.assertTrue(input_a.shape[:2] == (batch_size, pad_max_length)   )

      if feature_size > 1:
       self.assertTrue(input_a.shape[2] == input_a.shape[2] == feature_size   )

      # test padding for `pad_to_multiple_of` for List[int] + numpy
      UpperCamelCase_     :					Union[str, Any]   	=feat_extract.pad(_lowerCamelCase      ,  pad_to_multiple_of=10   )
      UpperCamelCase_     :					Tuple   	=input_a[input_name]

      UpperCamelCase_     :					List[Any]   	=feat_extract.pad(_lowerCamelCase      ,  padding='longest'      ,  pad_to_multiple_of=10   )
      UpperCamelCase_     :					Optional[Any]   	=input_a[input_name]

      UpperCamelCase_     :					int   	=feat_extract.pad(
          _lowerCamelCase      ,  padding='max_length'      ,  pad_to_multiple_of=10      ,  max_length=_lowerCamelCase   )
      UpperCamelCase_     :					List[Any]   	=input_a[input_name]

      UpperCamelCase_     :					Tuple   	=feat_extract.pad(
          _lowerCamelCase      ,  padding='max_length'      ,  pad_to_multiple_of=10      ,  max_length=_lowerCamelCase      ,  return_tensors='np'      ,  )
      UpperCamelCase_     :					Optional[Any]   	=input_a[input_name]

      self.assertTrue(all(len(_lowerCamelCase   ) % 10 == 0 for x in input_a   )   )
      self.assertTrue(_inputs_are_equal(_lowerCamelCase      ,  _lowerCamelCase   )   )

      UpperCamelCase_     :					Union[str, Any]   	=pad_max_length if pad_max_length % 10 == 0 else (pad_max_length // 10 + 1) * 10
      self.assertTrue(all(len(_lowerCamelCase   ) == expected_mult_pad_length for x in input_a   )   )
      self.assertEqual(input_a.shape[:2]      ,  (batch_size, expected_mult_pad_length)   )

      if feature_size > 1:
       self.assertTrue(input_a.shape[2] == feature_size   )

      # Check padding value is correct
      UpperCamelCase_     :					Union[str, Any]   	=(np.ones(self.feat_extract_tester.feature_size   ) * feat_extract.padding_value).sum()
      self.assertTrue(
          abs(np.asarray(input_a[0]   )[pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)   )
          < 1E-3   )
      self.assertTrue(
          abs(
              np.asarray(input_a[1]   )[pad_min_length + pad_diff :].sum()
              - padding_vector_sum * (pad_max_length - pad_min_length - pad_diff)   )
          < 1E-3   )
      self.assertTrue(
          abs(
              np.asarray(input_a[2]   )[pad_min_length + 2 * pad_diff :].sum()
              - padding_vector_sum * (pad_max_length - pad_min_length - 2 * pad_diff)   )
          < 1E-3   )
      self.assertTrue(
          abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)   ) < 1E-3   )
      self.assertTrue(
          abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (expected_mult_pad_length - pad_min_length)   )
          < 1E-3   )



     def  lowerCamelCase_						(							self :str      ,  _lowerCamelCase :Optional[Any]=False   ):







      '''simple docstring'''



      def _inputs_have_equal_length(_lowerCamelCase :Optional[int]   ):
       UpperCamelCase_     :					Optional[int]   	=len(input[0]   )
       for input_slice in input[1:]:
        if len(_lowerCamelCase   ) != length:
         return False
       return True

      def _inputs_are_equal(_lowerCamelCase :Tuple      ,  _lowerCamelCase :int   ):
       if len(_lowerCamelCase   ) != len(_lowerCamelCase   ):
        return False

       for input_slice_a, input_slice_a in zip(_lowerCamelCase      ,  _lowerCamelCase   ):
        if not np.allclose(np.asarray(_lowerCamelCase   )      ,  np.asarray(_lowerCamelCase   )      ,  atol=1E-3   ):
         return False
       return True

      UpperCamelCase_     :					Tuple   	=self.feature_extraction_class(**self.feat_extract_dict   )
      UpperCamelCase_     :					List[str]   	=self.feat_extract_tester.prepare_inputs_for_common(numpify=_lowerCamelCase   )
      UpperCamelCase_     :					Optional[Any]   	=feat_extract.model_input_names[0]

      UpperCamelCase_     :					Optional[int]   	=BatchFeature({input_name: speech_inputs}   )

      # truncate to smallest
      UpperCamelCase_     :					str   	=feat_extract.pad(
          _lowerCamelCase      ,  padding='max_length'      ,  max_length=len(speech_inputs[0]   )      ,  truncation=_lowerCamelCase   )
      UpperCamelCase_     :					List[str]   	=input_a[input_name]

      UpperCamelCase_     :					Dict   	=feat_extract.pad(_lowerCamelCase      ,  padding='max_length'      ,  max_length=len(speech_inputs[0]   )   )
      UpperCamelCase_     :					Any   	=input_a[input_name]

      self.assertTrue(_inputs_have_equal_length(_lowerCamelCase   )   )
      self.assertFalse(_inputs_have_equal_length(_lowerCamelCase   )   )

      # truncate to smallest with np
      UpperCamelCase_     :					Optional[Any]   	=feat_extract.pad(
          _lowerCamelCase      ,  padding='max_length'      ,  max_length=len(speech_inputs[0]   )      ,  return_tensors='np'      ,  truncation=_lowerCamelCase      ,  )
      UpperCamelCase_     :					Dict   	=input_a[input_name]

      UpperCamelCase_     :					Optional[int]   	=feat_extract.pad(
          _lowerCamelCase      ,  padding='max_length'      ,  max_length=len(speech_inputs[0]   )      ,  return_tensors='np'   )
      UpperCamelCase_     :					Optional[int]   	=input_a[input_name]

      self.assertTrue(_inputs_have_equal_length(_lowerCamelCase   )   )
      self.assertTrue(input_a.shape[1] == len(speech_inputs[0]   )   )

      # since truncation forces padding to be smaller than longest input
      # function can't return `np.ndarray`, but has to return list
      self.assertFalse(_inputs_have_equal_length(_lowerCamelCase   )   )

      # truncate to middle
      UpperCamelCase_     :					Optional[int]   	=feat_extract.pad(
          _lowerCamelCase      ,  padding='max_length'      ,  max_length=len(speech_inputs[1]   )      ,  truncation=_lowerCamelCase      ,  return_tensors='np'      ,  )
      UpperCamelCase_     :					Tuple   	=input_a[input_name]

      UpperCamelCase_     :					Optional[Any]   	=feat_extract.pad(
          _lowerCamelCase      ,  padding='max_length'      ,  max_length=len(speech_inputs[1]   )      ,  truncation=_lowerCamelCase   )
      UpperCamelCase_     :					List[Any]   	=input_a[input_name]

      UpperCamelCase_     :					Dict   	=feat_extract.pad(
          _lowerCamelCase      ,  padding='max_length'      ,  max_length=len(speech_inputs[1]   )      ,  return_tensors='np'   )
      UpperCamelCase_     :					List[str]   	=input_a[input_name]

      self.assertTrue(input_a.shape[1] == len(speech_inputs[1]   )   )
      self.assertTrue(_inputs_have_equal_length(_lowerCamelCase   )   )
      self.assertTrue(_inputs_have_equal_length(_lowerCamelCase   )   )
      self.assertTrue(_inputs_are_equal(_lowerCamelCase      ,  _lowerCamelCase   )   )

      # since truncation forces padding to be smaller than longest input
      # function can't return `np.ndarray`, but has to return list
      self.assertFalse(_inputs_have_equal_length(_lowerCamelCase   )   )
      self.assertTrue(len(input_a[-1]   ) == len(speech_inputs[-1]   )   )

      # padding has to be max_length when setting `truncation=True`
      with self.assertRaises(_lowerCamelCase   ):
       feat_extract.pad(_lowerCamelCase      ,  truncation=_lowerCamelCase   )[input_name]

      # padding has to be max_length when setting `truncation=True`
      with self.assertRaises(_lowerCamelCase   ):
       feat_extract.pad(_lowerCamelCase      ,  padding='longest'      ,  truncation=_lowerCamelCase   )[input_name]

      # padding has to be max_length when setting `truncation=True`
      with self.assertRaises(_lowerCamelCase   ):
       feat_extract.pad(_lowerCamelCase      ,  padding='longest'      ,  truncation=_lowerCamelCase   )[input_name]

      # max_length parameter has to be provided when setting `truncation=True` and padding="max_length"
      with self.assertRaises(_lowerCamelCase   ):
       feat_extract.pad(_lowerCamelCase      ,  padding='max_length'      ,  truncation=_lowerCamelCase   )[input_name]

      # test truncation for `pad_to_multiple_of` for List[int] + numpy
      UpperCamelCase_     :					Dict   	=12
      UpperCamelCase_     :					Tuple   	=feat_extract.pad(
          _lowerCamelCase      ,  padding='max_length'      ,  max_length=len(speech_inputs[0]   )      ,  pad_to_multiple_of=_lowerCamelCase      ,  truncation=_lowerCamelCase      ,  )
      UpperCamelCase_     :					Tuple   	=input_a[input_name]

      UpperCamelCase_     :					Any   	=feat_extract.pad(
          _lowerCamelCase      ,  padding='max_length'      ,  max_length=len(speech_inputs[0]   )      ,  pad_to_multiple_of=_lowerCamelCase      ,  )
      UpperCamelCase_     :					Optional[int]   	=input_a[input_name]

      # retrieve expected_length as multiple of pad_to_multiple_of
      UpperCamelCase_     :					Optional[Any]   	=len(speech_inputs[0]   )
      if expected_length % pad_to_multiple_of != 0:
       UpperCamelCase_     :					Tuple   	=((len(speech_inputs[0]   ) // pad_to_multiple_of) + 1) * pad_to_multiple_of

      self.assertTrue(len(input_a[0]   ) == expected_length   )
      self.assertTrue(_inputs_have_equal_length(_lowerCamelCase   )   )
      self.assertFalse(_inputs_have_equal_length(_lowerCamelCase   )   )



     def  lowerCamelCase_						(							self :Optional[Any]   ):







      '''simple docstring'''



      self._check_padding(numpify=_lowerCamelCase   )



     def  lowerCamelCase_						(							self :List[Any]   ):







      '''simple docstring'''



      self._check_padding(numpify=_lowerCamelCase   )



     def  lowerCamelCase_						(							self :Optional[int]   ):







      '''simple docstring'''



      self._check_truncation(numpify=_lowerCamelCase   )



     def  lowerCamelCase_						(							self :int   ):







      '''simple docstring'''



      self._check_truncation(numpify=_lowerCamelCase   )



     @require_torch
     def  lowerCamelCase_						(							self :List[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[str]   	=self.feature_extraction_class(**self.feat_extract_dict   )
      UpperCamelCase_     :					Optional[Any]   	=self.feat_extract_tester.prepare_inputs_for_common()
      UpperCamelCase_     :					List[str]   	=feat_extract.model_input_names[0]

      UpperCamelCase_     :					Optional[Any]   	=BatchFeature({input_name: speech_inputs}   )

      UpperCamelCase_     :					Any   	=feat_extract.pad(_lowerCamelCase      ,  padding='longest'      ,  return_tensors='np'   )[input_name]
      UpperCamelCase_     :					Optional[int]   	=feat_extract.pad(_lowerCamelCase      ,  padding='longest'      ,  return_tensors='pt'   )[input_name]

      self.assertTrue(abs(input_np.astype(np.floataa   ).sum() - input_pt.numpy().astype(np.floataa   ).sum()   ) < 1E-2   )



     @require_tf
     def  lowerCamelCase_						(							self :List[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[str]   	=self.feature_extraction_class(**self.feat_extract_dict   )
      UpperCamelCase_     :					Tuple   	=self.feat_extract_tester.prepare_inputs_for_common()
      UpperCamelCase_     :					int   	=feat_extract.model_input_names[0]

      UpperCamelCase_     :					List[Any]   	=BatchFeature({input_name: speech_inputs}   )

      UpperCamelCase_     :					str   	=feat_extract.pad(_lowerCamelCase      ,  padding='longest'      ,  return_tensors='np'   )[input_name]
      UpperCamelCase_     :					int   	=feat_extract.pad(_lowerCamelCase      ,  padding='longest'      ,  return_tensors='tf'   )[input_name]

      self.assertTrue(abs(input_np.astype(np.floataa   ).sum() - input_tf.numpy().astype(np.floataa   ).sum()   ) < 1E-2   )



     def  lowerCamelCase_						(							self :Any   ):







      '''simple docstring'''



      UpperCamelCase_     :					Tuple   	=self.feat_extract_dict
      UpperCamelCase_     :					Dict   	=True
      UpperCamelCase_     :					str   	=self.feature_extraction_class(**_lowerCamelCase   )
      UpperCamelCase_     :					Optional[Any]   	=self.feat_extract_tester.prepare_inputs_for_common()
      UpperCamelCase_     :					Optional[Any]   	=[len(_lowerCamelCase   ) for x in speech_inputs]
      UpperCamelCase_     :					List[Any]   	=feat_extract.model_input_names[0]

      UpperCamelCase_     :					int   	=BatchFeature({input_name: speech_inputs}   )

      UpperCamelCase_     :					List[str]   	=feat_extract.pad(_lowerCamelCase      ,  padding='longest'      ,  return_tensors='np'   )
      self.assertIn('attention_mask'      ,  _lowerCamelCase   )
      self.assertListEqual(list(processed.attention_mask.shape   )      ,  list(processed[input_name].shape[:2]   )   )
      self.assertListEqual(processed.attention_mask.sum(-1   ).tolist()      ,  _lowerCamelCase   )




     def  lowerCamelCase_						(							self :Tuple   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[str]   	=self.feat_extract_dict
      UpperCamelCase_     :					Dict   	=True
      UpperCamelCase_     :					Any   	=self.feature_extraction_class(**_lowerCamelCase   )
      UpperCamelCase_     :					Union[str, Any]   	=self.feat_extract_tester.prepare_inputs_for_common()
      UpperCamelCase_     :					Dict   	=[len(_lowerCamelCase   ) for x in speech_inputs]
      UpperCamelCase_     :					Tuple   	=feat_extract.model_input_names[0]

      UpperCamelCase_     :					List[Any]   	=BatchFeature({input_name: speech_inputs}   )
      UpperCamelCase_     :					str   	=min(_lowerCamelCase   )

      UpperCamelCase_     :					str   	=feat_extract.pad(
          _lowerCamelCase      ,  padding='max_length'      ,  max_length=_lowerCamelCase      ,  truncation=_lowerCamelCase      ,  return_tensors='np'   )
      self.assertIn('attention_mask'      ,  _lowerCamelCase   )
      self.assertListEqual(
          list(processed_pad.attention_mask.shape   )      ,  [processed_pad[input_name].shape[0], max_length]   )
      self.assertListEqual(
          processed_pad.attention_mask[:, :max_length].sum(-1   ).tolist()      ,  [max_length for x in speech_inputs]   )


"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							=     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__SCREAMING_SNAKE_CASE							=     [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

__SCREAMING_SNAKE_CASE							=     {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
__SCREAMING_SNAKE_CASE							=     {F"""funnel-transformer/{name}""": 512 for name in _model_names}
__SCREAMING_SNAKE_CASE							=     {F"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}





class 				a__					(						A__       ):
     UpperCAmelCase__         =      VOCAB_FILES_NAMES
     UpperCAmelCase__         =      PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__         =      PRETRAINED_INIT_CONFIGURATION
     UpperCAmelCase__         =      FunnelTokenizer
     UpperCAmelCase__         =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__         =      2



     def __init__(							self :str      ,  _lowerCamelCase :int=None      ,  _lowerCamelCase :List[Any]=None      ,  _lowerCamelCase :Dict=True      ,  _lowerCamelCase :Optional[Any]="<unk>"      ,  _lowerCamelCase :Optional[Any]="<sep>"      ,  _lowerCamelCase :List[Any]="<pad>"      ,  _lowerCamelCase :List[Any]="<cls>"      ,  _lowerCamelCase :Union[str, Any]="<mask>"      ,  _lowerCamelCase :Any="<s>"      ,  _lowerCamelCase :Union[str, Any]="</s>"      ,  _lowerCamelCase :str=True      ,  _lowerCamelCase :Dict=True      ,  _lowerCamelCase :Tuple=None      ,  _lowerCamelCase :Tuple="##"      ,  **_lowerCamelCase :int      ,  ):







      '''simple docstring'''



      super().__init__(
          _lowerCamelCase      ,  tokenizer_file=_lowerCamelCase      ,  do_lower_case=_lowerCamelCase      ,  unk_token=_lowerCamelCase      ,  sep_token=_lowerCamelCase      ,  pad_token=_lowerCamelCase      ,  cls_token=_lowerCamelCase      ,  mask_token=_lowerCamelCase      ,  bos_token=_lowerCamelCase      ,  eos_token=_lowerCamelCase      ,  clean_text=_lowerCamelCase      ,  tokenize_chinese_chars=_lowerCamelCase      ,  strip_accents=_lowerCamelCase      ,  wordpieces_prefix=_lowerCamelCase      ,  **_lowerCamelCase      ,  )

      UpperCamelCase_     :					str   	=json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
      if (
          normalizer_state.get('lowercase'      ,  _lowerCamelCase   ) != do_lower_case
          or normalizer_state.get('strip_accents'      ,  _lowerCamelCase   ) != strip_accents
          or normalizer_state.get('handle_chinese_chars'      ,  _lowerCamelCase   ) != tokenize_chinese_chars
      ):
       UpperCamelCase_     :					int   	=getattr(_lowerCamelCase      ,  normalizer_state.pop('type'   )   )
       UpperCamelCase_     :					Dict   	=do_lower_case
       UpperCamelCase_     :					Tuple   	=strip_accents
       UpperCamelCase_     :					int   	=tokenize_chinese_chars
       UpperCamelCase_     :					Tuple   	=normalizer_class(**_lowerCamelCase   )

      UpperCamelCase_     :					Dict   	=do_lower_case



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[int]=None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output



     def  lowerCamelCase_						(							self :List[Any]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[int]   	=[self.sep_token_id]
      UpperCamelCase_     :					List[Any]   	=[self.cls_token_id]
      if token_ids_a is None:
       return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0]
      return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]




     def  lowerCamelCase_						(							self :Optional[Any]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[str] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=self._tokenizer.model.save(_lowerCamelCase      ,  name=_lowerCamelCase   )
      return tuple(_lowerCamelCase   )
