


"""simple docstring"""


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_snake_case        =				logging.get_logger(__name__)

_snake_case        =				{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_snake_case        =				[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

_snake_case        =				{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
_snake_case        =				{f"""funnel-transformer/{name}""": 512 for name in _model_names}
_snake_case        =				{f"""funnel-transformer/{name}""": {"do_lower_case": True} for name in _model_names}



class 	_a   (   SCREAMING_SNAKE_CASE_			):
    a_     :					List[str]   							=     VOCAB_FILES_NAMES
    a_     :					Optional[Any]   							=     PRETRAINED_VOCAB_FILES_MAP
    a_     :					Optional[Any]   							=     PRETRAINED_INIT_CONFIGURATION
    a_     :					Optional[Any]   							=     FunnelTokenizer
    a_     :					Optional[Any]   							=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a_     :					Dict   							=     2
    def __init__(			self				:       Optional[Any]				,       SCREAMING_SNAKE_CASE__				:       List[Any]=None				,       SCREAMING_SNAKE_CASE__				:       Optional[int]=None				,       SCREAMING_SNAKE_CASE__				:       Optional[Any]=True				,       SCREAMING_SNAKE_CASE__				:       Tuple="<unk>"				,       SCREAMING_SNAKE_CASE__				:       int="<sep>"				,       SCREAMING_SNAKE_CASE__				:       List[Any]="<pad>"				,       SCREAMING_SNAKE_CASE__				:       List[Any]="<cls>"				,       SCREAMING_SNAKE_CASE__				:       Any="<mask>"				,       SCREAMING_SNAKE_CASE__				:       Tuple="<s>"				,       SCREAMING_SNAKE_CASE__				:       List[Any]="</s>"				,       SCREAMING_SNAKE_CASE__				:       Optional[int]=True				,       SCREAMING_SNAKE_CASE__				:       str=True				,       SCREAMING_SNAKE_CASE__				:       Optional[int]=None				,       SCREAMING_SNAKE_CASE__				:       List[str]="##"				,       **SCREAMING_SNAKE_CASE__				:       Any				,       ):
          super().__init__(
              _a				,       tokenizer_file=_a				,       do_lower_case=_a				,       unk_token=_a				,       sep_token=_a				,       pad_token=_a				,       cls_token=_a				,       mask_token=_a				,       bos_token=_a				,       eos_token=_a				,       clean_text=_a				,       tokenize_chinese_chars=_a				,       strip_accents=_a				,       wordpieces_prefix=_a				,       **_a				,       )

          lowerCamelCase__							     =		json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
          if (
              normalizer_state.get('lowercase'				,       _a					) != do_lower_case
              or normalizer_state.get('strip_accents'				,       _a					) != strip_accents
              or normalizer_state.get('handle_chinese_chars'				,       _a					) != tokenize_chinese_chars
          ):
                lowerCamelCase__							     =		getattr(_a				,       normalizer_state.pop('type'					)					)
                lowerCamelCase__							     =		do_lower_case
                lowerCamelCase__							     =		strip_accents
                lowerCamelCase__							     =		tokenize_chinese_chars
                lowerCamelCase__							     =		normalizer_class(**_a					)

          lowerCamelCase__							     =		do_lower_case
    def   _UpperCamelCase							(			self				:       Optional[Any]				,       SCREAMING_SNAKE_CASE__				:       List[Any]				,       SCREAMING_SNAKE_CASE__				:       int=None					):
          lowerCamelCase__							     =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

          return output
    def   _UpperCamelCase							(			self				:       Optional[Any]				,       SCREAMING_SNAKE_CASE__				:       List[int]				,       SCREAMING_SNAKE_CASE__				:       Optional[List[int]] = None					):
          lowerCamelCase__							     =		[self.sep_token_id]
          lowerCamelCase__							     =		[self.cls_token_id]
          if token_ids_a is None:
                return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0]
          return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]


    def   _UpperCamelCase							(			self				:       int				,       SCREAMING_SNAKE_CASE__				:       str				,       SCREAMING_SNAKE_CASE__				:       Optional[str] = None					):
          lowerCamelCase__							     =		self._tokenizer.model.save(_a				,       name=_a					)
          return tuple(_a					)








'''simple docstring'''





import json
import os
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase											=		logging.get_logger(__name__)

_UpperCamelCase											=		{'vocab_file': 'vocab.json'}

_UpperCamelCase											=		{
    'vocab_file': {
        'mgp-str': 'https://huggingface.co/alibaba-damo/mgp-str-base/blob/main/vocab.json',
    }
}

_UpperCamelCase											=		{'mgp-str': 27}






class    lowerCamelCase_       (							SCREAMING_SNAKE_CASE_	):
  """simple docstring"""




  a_				   =VOCAB_FILES_NAMES
  a_				   =PRETRAINED_VOCAB_FILES_MAP
  a_				   =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




  def __init__(					self :      Optional[Any]     ,      _a :      Optional[Any]     ,      _a :      Any="[GO]"     ,      _a :      Optional[Any]="[GO]"     ,      _a :      Optional[int]="[s]"     ,      _a :      Optional[int]="[GO]"     ,      **_a :      Union[str, Any]  )   ->   Optional[int]:
    super().__init__(
        unk_token=_a     ,      bos_token=_a     ,      eos_token=_a     ,      pad_token=_a     ,      **_a     ,      )

    with open(_a     ,      encoding='utf-8'  ) as vocab_handle:
      __lowerCamelCase			:			List[str]       			=					json.load(_a  )
    __lowerCamelCase			:			Tuple       			=					{v: k for k, v in self.vocab.items()}




  @property
  def 		_lowercase	(					self :      Union[str, Any]  )   ->   List[str]:
    return len(self.vocab  )




  def 		_lowercase	(					self :      Union[str, Any]  )   ->   Tuple:
    return dict(self.vocab     ,      **self.added_tokens_encoder  )




  def 		_lowercase	(					self :      Dict     ,      _a :      Tuple  )   ->   int:
    __lowerCamelCase			:			Optional[Any]       			=					[]
    for s in text:
      char_tokens.extend(_a  )
    return char_tokens




  def 		_lowercase	(					self :      Optional[Any]     ,      _a :      Any  )   ->   Dict:
    return self.vocab.get(_a     ,      self.vocab.get(self.unk_token  )  )




  def 		_lowercase	(					self :      List[Any]     ,      _a :      Dict  )   ->   Optional[int]:
    return self.decoder.get(_a  )




  def 		_lowercase	(					self :      Union[str, Any]     ,      _a :      str     ,      _a :      Optional[str] = None  )   ->   Tuple[str]:
    if not os.path.isdir(_a  ):
      logger.error('Vocabulary path ({}) should be a directory'.format(_a  )  )
      return
    __lowerCamelCase			:			List[Any]       			=					os.path.join(
        _a     ,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )

    with open(_a     ,      'w'     ,      encoding='utf-8'  ) as f:
      f.write(json.dumps(self.vocab     ,      indent=2     ,      sort_keys=_a     ,      ensure_ascii=_a  ) + '\n'  )

    return (vocab_file,)






