



'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_lowerCAmelCase 		=      logging.get_logger(__name__)

_lowerCAmelCase 		=      {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_lowerCAmelCase 		=      [
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

_lowerCAmelCase 		=      {
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
_lowerCAmelCase 		=      {F"""funnel-transformer/{name}""": 512 for name in _model_names}
_lowerCAmelCase 		=      {F"""funnel-transformer/{name}""": {'''do_lower_case''': True} for name in _model_names}






class 			lowerCAmelCase_(							SCREAMING_SNAKE_CASE_	):
  '''simple docstring'''




  __lowercase				: Optional[int]           = VOCAB_FILES_NAMES
  __lowercase				: Union[str, Any]           = PRETRAINED_VOCAB_FILES_MAP
  __lowercase				: Union[str, Any]           = PRETRAINED_INIT_CONFIGURATION
  __lowercase				: Tuple           = FunnelTokenizer
  __lowercase				: Any           = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowercase				: Optional[Any]           = 2




  def __init__(       self    ,__UpperCAmelCase=None    ,__UpperCAmelCase=None    ,__UpperCAmelCase=True    ,__UpperCAmelCase="<unk>"    ,__UpperCAmelCase="<sep>"    ,__UpperCAmelCase="<pad>"    ,__UpperCAmelCase="<cls>"    ,__UpperCAmelCase="<mask>"    ,__UpperCAmelCase="<s>"    ,__UpperCAmelCase="</s>"    ,__UpperCAmelCase=True    ,__UpperCAmelCase=True    ,__UpperCAmelCase=None    ,__UpperCAmelCase="##"    ,**__UpperCAmelCase    ,)       ->						Dict:
      super().__init__(
          __SCREAMING_SNAKE_CASE    ,tokenizer_file=__SCREAMING_SNAKE_CASE    ,do_lower_case=__SCREAMING_SNAKE_CASE    ,unk_token=__SCREAMING_SNAKE_CASE    ,sep_token=__SCREAMING_SNAKE_CASE    ,pad_token=__SCREAMING_SNAKE_CASE    ,cls_token=__SCREAMING_SNAKE_CASE    ,mask_token=__SCREAMING_SNAKE_CASE    ,bos_token=__SCREAMING_SNAKE_CASE    ,eos_token=__SCREAMING_SNAKE_CASE    ,clean_text=__SCREAMING_SNAKE_CASE    ,tokenize_chinese_chars=__SCREAMING_SNAKE_CASE    ,strip_accents=__SCREAMING_SNAKE_CASE    ,wordpieces_prefix=__SCREAMING_SNAKE_CASE    ,**__SCREAMING_SNAKE_CASE    ,)

      lowerCAmelCase__					:    Any				 =   json.loads(self.backend_tokenizer.normalizer.__getstate__() )
      if (
          normalizer_state.get("""lowercase"""    ,__SCREAMING_SNAKE_CASE ) != do_lower_case
          or normalizer_state.get("""strip_accents"""    ,__SCREAMING_SNAKE_CASE ) != strip_accents
          or normalizer_state.get("""handle_chinese_chars"""    ,__SCREAMING_SNAKE_CASE ) != tokenize_chinese_chars
      ):
          lowerCAmelCase__					:    List[str]				 =   getattr(__SCREAMING_SNAKE_CASE    ,normalizer_state.pop("""type""" ) )
          lowerCAmelCase__					:    int				 =   do_lower_case
          lowerCAmelCase__					:    Optional[int]				 =   strip_accents
          lowerCAmelCase__					:    Union[str, Any]				 =   tokenize_chinese_chars
          lowerCAmelCase__					:    Any				 =   normalizer_class(**__SCREAMING_SNAKE_CASE )

      lowerCAmelCase__					:    Optional[Any]				 =   do_lower_case




  def 				UpperCAmelCase_      (       self    ,__UpperCAmelCase    ,__UpperCAmelCase=None )       ->						List[str]:
      lowerCAmelCase__					:    Optional[int]				 =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
          output += token_ids_a + [self.sep_token_id]

      return output




  def 				UpperCAmelCase_      (       self    ,__UpperCAmelCase    ,__UpperCAmelCase = None )       ->						List[int]:
      lowerCAmelCase__					:    int				 =   [self.sep_token_id]
      lowerCAmelCase__					:    List[Any]				 =   [self.cls_token_id]
      if token_ids_a is None:
          return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0]
      return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






  def 				UpperCAmelCase_      (       self    ,__UpperCAmelCase    ,__UpperCAmelCase = None )       ->						Tuple[str]:
      lowerCAmelCase__					:    List[str]				 =   self._tokenizer.model.save(__SCREAMING_SNAKE_CASE    ,name=__SCREAMING_SNAKE_CASE )
      return tuple(__SCREAMING_SNAKE_CASE )










'''simple docstring'''







from __future__ import annotations





def 		_UpperCAmelCase (				__A				: str    ,		__A				: str   ):
				a_		:	int									=   get_failure_array(__A   )

				# 2) Step through text searching for pattern
				a_					,      a_		:	Any									=   0, 0  # index into text, pattern
				while i < len(__A   ):
								if pattern[j] == text[i]:
												if j == (len(__A   ) - 1):
																return True
												j += 1

								# if this is a prefix in our pattern
								# just go back far enough to continue
								elif j > 0:
												a_		:	Any									=   failure[j - 1]
												continue
								i += 1
				return False





def 		_UpperCAmelCase (				__A				: str   ):
				a_		:	Optional[Any]									=   [0]
				a_		:	Any									=   0
				a_		:	int									=   1
				while j < len(__A   ):
								if pattern[i] == pattern[j]:
												i += 1
								elif i > 0:
												a_		:	List[Any]									=   failure[i - 1]
												continue
								j += 1
								failure.append(__A   )
				return failure


if __name__ == "__main__":
		# Test 1)
		__lowerCAmelCase									=   'abc1abc12'
		__lowerCAmelCase									=   'alskfjaldsabc1abc1abc12k23adsfabcabc'
		__lowerCAmelCase									=   'alskfjaldsk23adsfabcabc'
		assert kmp(pattern, texta) and not kmp(pattern, texta)

		# Test 2)
		__lowerCAmelCase									=   'ABABX'
		__lowerCAmelCase									=   'ABABZABABYABABX'
		assert kmp(pattern, text)

		# Test 3)
		__lowerCAmelCase									=   'AAAB'
		__lowerCAmelCase									=   'ABAAAAAB'
		assert kmp(pattern, text)

		# Test 4)
		__lowerCAmelCase									=   'abcdabcy'
		__lowerCAmelCase									=   'abcxabcdabxabcdabcdabcy'
		assert kmp(pattern, text)

		# Test 5)
		__lowerCAmelCase									=   'aabaabaaa'
		assert get_failure_array(pattern) == [0, 1, 0, 1, 2, 3, 4, 5, 2]




