




'''simple docstring'''
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_SCREAMING_SNAKE_CASE			=						logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE			=						{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_SCREAMING_SNAKE_CASE			=						[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

_SCREAMING_SNAKE_CASE			=						{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
_SCREAMING_SNAKE_CASE			=						{f"""funnel-transformer/{name}""": 5_12 for name in _model_names}
_SCREAMING_SNAKE_CASE			=						{f"""funnel-transformer/{name}""": {"do_lower_case": True} for name in _model_names}
class   __UpperCAmelCase				(     lowerCAmelCase				):



					'''simple docstring'''





					_UpperCamelCase							=		VOCAB_FILES_NAMES
					_UpperCamelCase							=		PRETRAINED_VOCAB_FILES_MAP
					_UpperCamelCase							=		PRETRAINED_INIT_CONFIGURATION
					_UpperCamelCase							=		FunnelTokenizer
					_UpperCamelCase							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_UpperCamelCase							=		2



					def __init__(	self     :						Dict      ,       _lowercase     :						Optional[int]=None      ,       _lowercase     :						Dict=None      ,       _lowercase     :						str=True      ,       _lowercase     :						Optional[int]="<unk>"      ,       _lowercase     :						Optional[Any]="<sep>"      ,       _lowercase     :						List[str]="<pad>"      ,       _lowercase     :						Optional[Any]="<cls>"      ,       _lowercase     :						Optional[Any]="<mask>"      ,       _lowercase     :						Union[str, Any]="<s>"      ,       _lowercase     :						Union[str, Any]="</s>"      ,       _lowercase     :						Optional[Any]=True      ,       _lowercase     :						Union[str, Any]=True      ,       _lowercase     :						Dict=None      ,       _lowercase     :						Dict="##"      ,       **_lowercase     :						Dict      ,       )    ->							int:
								super().__init__(
								    _lowercase      ,       tokenizer_file=_lowercase      ,       do_lower_case=_lowercase      ,       unk_token=_lowercase      ,       sep_token=_lowercase      ,       pad_token=_lowercase      ,       cls_token=_lowercase      ,       mask_token=_lowercase      ,       bos_token=_lowercase      ,       eos_token=_lowercase      ,       clean_text=_lowercase      ,       tokenize_chinese_chars=_lowercase      ,       strip_accents=_lowercase      ,       wordpieces_prefix=_lowercase      ,       **_lowercase      ,       )

								A_									= json.loads(self.backend_tokenizer.normalizer.__getstate__())
								if (
								    normalizer_state.get('lowercase'      ,       _lowercase) != do_lower_case
								    or normalizer_state.get('strip_accents'      ,       _lowercase) != strip_accents
								    or normalizer_state.get('handle_chinese_chars'      ,       _lowercase) != tokenize_chinese_chars
								):
											A_									= getattr(_lowercase      ,       normalizer_state.pop('type'))
											A_									= do_lower_case
											A_									= strip_accents
											A_									= tokenize_chinese_chars
											A_									= normalizer_class(**_lowercase)

								A_									= do_lower_case



					def     __snake_case						(	self     :						Optional[Any]      ,       _lowercase     :						str      ,       _lowercase     :						Dict=None)    ->							Optional[int]:
								A_									= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
											output += token_ids_a + [self.sep_token_id]

								return output



					def     __snake_case						(	self     :						Any      ,       _lowercase     :						List[int]      ,       _lowercase     :						Optional[List[int]] = None)    ->							List[int]:
								A_									= [self.sep_token_id]
								A_									= [self.cls_token_id]
								if token_ids_a is None:
											return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0]
								return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



					def     __snake_case						(	self     :						Optional[int]      ,       _lowercase     :						str      ,       _lowercase     :						Optional[str] = None)    ->							Tuple[str]:
								A_									= self._tokenizer.model.save(_lowercase      ,       name=_lowercase)
								return tuple(_lowercase)










'''simple docstring'''
import os
import sys

_SCREAMING_SNAKE_CASE			=						os.path.join(os.path.dirname(__file__), "src")
sys.path.append(SRC_DIR)


from transformers import (
    AutoConfig,
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForMaskedLM,
    AutoModelForQuestionAnswering,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    add_start_docstrings,
)


_SCREAMING_SNAKE_CASE			=						[
    "torch",
    "numpy",
    "tokenizers",
    "filelock",
    "requests",
    "tqdm",
    "regex",
    "sentencepiece",
    "sacremoses",
    "importlib_metadata",
    "huggingface_hub",
]





@add_start_docstrings(AutoConfig.__doc__     )
def 			lowerCamelCase(							*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )		->		int:
			return AutoConfig.from_pretrained(*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )





@add_start_docstrings(AutoTokenizer.__doc__     )
def 			lowerCamelCase(							*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )		->		List[str]:
			return AutoTokenizer.from_pretrained(*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )





@add_start_docstrings(AutoModel.__doc__     )
def 			lowerCamelCase(							*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )		->		Union[str, Any]:
			return AutoModel.from_pretrained(*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )





@add_start_docstrings(AutoModelForCausalLM.__doc__     )
def 			lowerCamelCase(							*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )		->		Optional[int]:
			return AutoModelForCausalLM.from_pretrained(*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )





@add_start_docstrings(AutoModelForMaskedLM.__doc__     )
def 			lowerCamelCase(							*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )		->		Dict:
			return AutoModelForMaskedLM.from_pretrained(*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )





@add_start_docstrings(AutoModelForSequenceClassification.__doc__     )
def 			lowerCamelCase(							*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )		->		Union[str, Any]:
			return AutoModelForSequenceClassification.from_pretrained(*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )






@add_start_docstrings(AutoModelForQuestionAnswering.__doc__     )
def 			lowerCamelCase(							*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )		->		str:
			return AutoModelForQuestionAnswering.from_pretrained(*SCREAMING_SNAKE_CASE_       ,**SCREAMING_SNAKE_CASE_     )




