



"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCAmelCase__              =							logging.get_logger(__name__)

lowerCAmelCase__              =							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase__              =							[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

lowerCAmelCase__              =							{
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCAmelCase__              =							{F"""funnel-transformer/{name}""": 512 for name in _model_names}
lowerCAmelCase__              =							{F"""funnel-transformer/{name}""": {'''do_lower_case''': True} for name in _model_names}






class 			__snake_case			(      A__):
  snake_case__							:  int         =		VOCAB_FILES_NAMES
  snake_case__							:  List[str]         =		PRETRAINED_VOCAB_FILES_MAP
  snake_case__							:  Any         =		PRETRAINED_INIT_CONFIGURATION
  snake_case__							:  List[Any]         =		FunnelTokenizer
  snake_case__							:  List[Any]         =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  snake_case__							:  List[str]         =		2




  def __init__(					self	:		str		,   __lowerCAmelCase	:		int=None		,   __lowerCAmelCase	:		List[Any]=None		,   __lowerCAmelCase	:		Dict=True		,   __lowerCAmelCase	:		Optional[Any]="<unk>"		,   __lowerCAmelCase	:		Optional[Any]="<sep>"		,   __lowerCAmelCase	:		List[Any]="<pad>"		,   __lowerCAmelCase	:		List[Any]="<cls>"		,   __lowerCAmelCase	:		Union[str, Any]="<mask>"		,   __lowerCAmelCase	:		Any="<s>"		,   __lowerCAmelCase	:		Union[str, Any]="</s>"		,   __lowerCAmelCase	:		str=True		,   __lowerCAmelCase	:		Dict=True		,   __lowerCAmelCase	:		Tuple=None		,   __lowerCAmelCase	:		Tuple="##"		,   **__lowerCAmelCase	:		int		,   ):






         """simple docstring"""






         super().__init__(
             _lowerCamelCase		,   tokenizer_file=_lowerCamelCase		,   do_lower_case=_lowerCamelCase		,   unk_token=_lowerCamelCase		,   sep_token=_lowerCamelCase		,   pad_token=_lowerCamelCase		,   cls_token=_lowerCamelCase		,   mask_token=_lowerCamelCase		,   bos_token=_lowerCamelCase		,   eos_token=_lowerCamelCase		,   clean_text=_lowerCamelCase		,   tokenize_chinese_chars=_lowerCamelCase		,   strip_accents=_lowerCamelCase		,   wordpieces_prefix=_lowerCamelCase		,   **_lowerCamelCase		,   )

         _lowerCamelCase		:							str            =					json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
         if (
             normalizer_state.get('''lowercase'''		,   _lowerCamelCase						) != do_lower_case
             or normalizer_state.get('''strip_accents'''		,   _lowerCamelCase						) != strip_accents
             or normalizer_state.get('''handle_chinese_chars'''		,   _lowerCamelCase						) != tokenize_chinese_chars
         ):
                _lowerCamelCase		:							int            =					getattr(_lowerCamelCase		,   normalizer_state.pop('''type'''						)						)
                _lowerCamelCase		:							Dict            =					do_lower_case
                _lowerCamelCase		:							Tuple            =					strip_accents
                _lowerCamelCase		:							int            =					tokenize_chinese_chars
                _lowerCamelCase		:							Tuple            =					normalizer_class(**_lowerCamelCase						)

         _lowerCamelCase		:							Dict            =					do_lower_case




  def 		SCREAMING_SNAKE_CASE							(					self	:		Optional[int]		,   __lowerCAmelCase	:		str		,   __lowerCAmelCase	:		Optional[int]=None						):






         """simple docstring"""






         _lowerCamelCase		:							Optional[Any]            =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

         return output




  def 		SCREAMING_SNAKE_CASE							(					self	:		List[Any]		,   __lowerCAmelCase	:		List[int]		,   __lowerCAmelCase	:		Optional[List[int]] = None						):






         """simple docstring"""






         _lowerCamelCase		:							Optional[int]            =					[self.sep_token_id]
         _lowerCamelCase		:							List[Any]            =					[self.cls_token_id]
         if token_ids_a is None:
                return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0]
         return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]




  def 		SCREAMING_SNAKE_CASE							(					self	:		Optional[Any]		,   __lowerCAmelCase	:		str		,   __lowerCAmelCase	:		Optional[str] = None						):






         """simple docstring"""






         _lowerCamelCase		:							int            =					self._tokenizer.model.save(_lowerCamelCase		,   name=_lowerCamelCase						)
         return tuple(_lowerCamelCase						)




"""simple docstring"""






from collections.abc import Callable





class 				a__					:



     def __init__(							self :Tuple      ,  _lowerCamelCase :Callable | None = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					list   	=[]
      # Stores indexes of each item for supporting updates and deletion.
      UpperCamelCase_     :					dict   	={}
      # Stores current size of heap.
      UpperCamelCase_     :					Any   	=0
      # Stores function used to evaluate the score of an item on which basis ordering
      # will be done.
      UpperCamelCase_     :					List[str]   	=key or (lambda _lowerCamelCase   : x)



     def  lowerCamelCase_						(							self :Tuple      ,  _lowerCamelCase :int   ):







      '''simple docstring'''



      return int((i - 1) / 2   ) if i > 0 else None



     def  lowerCamelCase_						(							self :Optional[Any]      ,  _lowerCamelCase :int   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[str]   	=int(2 * i + 1   )
      return left if 0 < left < self.size else None



     def  lowerCamelCase_						(							self :Tuple      ,  _lowerCamelCase :int   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=int(2 * i + 2   )
      return right if 0 < right < self.size else None



     def  lowerCamelCase_						(							self :Dict      ,  _lowerCamelCase :int      ,  _lowerCamelCase :int   ):







      '''simple docstring'''



      UpperCamelCase_	,     UpperCamelCase_     :					Optional[int]   	=(
          self.pos_map[self.arr[j][0]],
          self.pos_map[self.arr[i][0]],
      )
      # Then swap the items in the list.
      UpperCamelCase_	,     UpperCamelCase_     :					Union[str, Any]   	=self.arr[j], self.arr[i]



     def  lowerCamelCase_						(							self :Optional[Any]      ,  _lowerCamelCase :int      ,  _lowerCamelCase :int   ):







      '''simple docstring'''



      return self.arr[i][1] < self.arr[j][1]



     def  lowerCamelCase_						(							self :Any      ,  _lowerCamelCase :int   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=self._left(_lowerCamelCase   )
      UpperCamelCase_     :					List[Any]   	=self._right(_lowerCamelCase   )
      UpperCamelCase_     :					Optional[Any]   	=i

      if left is not None and not self._cmp(_lowerCamelCase      ,  _lowerCamelCase   ):
       UpperCamelCase_     :					Optional[int]   	=left
      if right is not None and not self._cmp(_lowerCamelCase      ,  _lowerCamelCase   ):
       UpperCamelCase_     :					List[Any]   	=right

      return valid_parent



     def  lowerCamelCase_						(							self :Any      ,  _lowerCamelCase :int   ):







      '''simple docstring'''



      UpperCamelCase_     :					Dict   	=self._parent(_lowerCamelCase   )
      while parent is not None and not self._cmp(_lowerCamelCase      ,  _lowerCamelCase   ):
       self._swap(_lowerCamelCase      ,  _lowerCamelCase   )
       UpperCamelCase_	,     UpperCamelCase_     :					Dict   	=parent, self._parent(_lowerCamelCase   )



     def  lowerCamelCase_						(							self :List[str]      ,  _lowerCamelCase :int   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=self._get_valid_parent(_lowerCamelCase   )
      while valid_parent != index:
       self._swap(_lowerCamelCase      ,  _lowerCamelCase   )
       UpperCamelCase_	,     UpperCamelCase_     :					int   	=valid_parent, self._get_valid_parent(_lowerCamelCase   )



     def  lowerCamelCase_						(							self :Optional[Any]      ,  _lowerCamelCase :int      ,  _lowerCamelCase :int   ):







      '''simple docstring'''



      if item not in self.pos_map:
       return
      UpperCamelCase_     :					List[Any]   	=self.pos_map[item]
      UpperCamelCase_     :					int   	=[item, self.key(_lowerCamelCase   )]
      # Make sure heap is right in both up and down direction.
      # Ideally only one of them will make any change.
      self._heapify_up(_lowerCamelCase   )
      self._heapify_down(_lowerCamelCase   )



     def  lowerCamelCase_						(							self :Tuple      ,  _lowerCamelCase :int   ):







      '''simple docstring'''



      if item not in self.pos_map:
       return
      UpperCamelCase_     :					Any   	=self.pos_map[item]
      del self.pos_map[item]
      UpperCamelCase_     :					Dict   	=self.arr[self.size - 1]
      UpperCamelCase_     :					Optional[int]   	=index
      self.size -= 1
      # Make sure heap is right in both up and down direction. Ideally only one
      # of them will make any change- so no performance loss in calling both.
      if self.size > index:
       self._heapify_up(_lowerCamelCase   )
       self._heapify_down(_lowerCamelCase   )



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :int      ,  _lowerCamelCase :int   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[int]   	=len(self.arr   )
      if arr_len == self.size:
       self.arr.append([item, self.key(_lowerCamelCase   )]   )
      else:
       UpperCamelCase_     :					str   	=[item, self.key(_lowerCamelCase   )]
      UpperCamelCase_     :					Optional[int]   	=self.size
      self.size += 1
      self._heapify_up(self.size - 1   )



     def  lowerCamelCase_						(							self :List[Any]   ):







      '''simple docstring'''



      return self.arr[0] if self.size else None




     def  lowerCamelCase_						(							self :Tuple   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=self.get_top()
      if top_item_tuple:
       self.delete_item(top_item_tuple[0]   )
      return top_item_tuple




def       A_  (				):
 pass


if __name__ == "__main__":
 import doctest

 doctest.testmod()
