




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


_lowerCamelCase			:	Union[str, Any]							 =     logging.get_logger(__name__)

_lowerCamelCase			:	Dict							 =     {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowerCamelCase			:	Tuple							 =     {
    """vocab_file""": {"""mobilebert-uncased""": """https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt"""},
    """tokenizer_file""": {
        """mobilebert-uncased""": """https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json"""
    },
}

_lowerCamelCase			:	Tuple							 =     {"""mobilebert-uncased""": 512}


_lowerCamelCase			:	int							 =     {}


class        UpperCamelCase_					(				UpperCAmelCase__  ):

    '''simple docstring'''






    UpperCAmelCase__													=   VOCAB_FILES_NAMES
    UpperCAmelCase__													=   PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase__													=   PRETRAINED_INIT_CONFIGURATION
    UpperCAmelCase__													=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase__													=   MobileBertTokenizer


    def __init__(    self   :			int      ,				UpperCAmelCase__   :			Any=None      ,				UpperCAmelCase__   :			Union[str, Any]=None      ,				UpperCAmelCase__   :			Dict=True      ,				UpperCAmelCase__   :			Dict="[UNK]"      ,				UpperCAmelCase__   :			Optional[int]="[SEP]"      ,				UpperCAmelCase__   :			Tuple="[PAD]"      ,				UpperCAmelCase__   :			Dict="[CLS]"      ,				UpperCAmelCase__   :			int="[MASK]"      ,				UpperCAmelCase__   :			Optional[int]=True      ,				UpperCAmelCase__   :			int=None      ,				**UpperCAmelCase__   :			Union[str, Any]      ,				)		->str:





           '''simple docstring'''

           super().__init__(
               UpperCAmelCase__      ,				tokenizer_file=UpperCAmelCase__      ,				do_lower_case=UpperCAmelCase__      ,				unk_token=UpperCAmelCase__      ,				sep_token=UpperCAmelCase__      ,				pad_token=UpperCAmelCase__      ,				cls_token=UpperCAmelCase__      ,				mask_token=UpperCAmelCase__      ,				tokenize_chinese_chars=UpperCAmelCase__      ,				strip_accents=UpperCAmelCase__      ,				**UpperCAmelCase__      ,				)

           A__   				=					json.loads(self.backend_tokenizer.normalizer.__getstate__())
           if (
               normalizer_state.get('''lowercase'''      ,				UpperCAmelCase__) != do_lower_case
               or normalizer_state.get('''strip_accents'''      ,				UpperCAmelCase__) != strip_accents
               or normalizer_state.get('''handle_chinese_chars'''      ,				UpperCAmelCase__) != tokenize_chinese_chars
           ):
                  A__   				=					getattr(UpperCAmelCase__      ,				normalizer_state.pop('''type'''))
                  A__   				=					do_lower_case
                  A__   				=					strip_accents
                  A__   				=					tokenize_chinese_chars
                  A__   				=					normalizer_class(**UpperCAmelCase__)

           A__   				=					do_lower_case


    def  SCREAMING_SNAKE_CASE							(    self   :			Optional[int]      ,				UpperCAmelCase__   :			Union[str, Any]      ,				UpperCAmelCase__   :			Tuple=None)		->Dict:





           '''simple docstring'''

           A__   				=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output


    def  SCREAMING_SNAKE_CASE							(    self   :			Dict      ,				UpperCAmelCase__   :			List[int]      ,				UpperCAmelCase__   :			Optional[List[int]] = None)		->List[int]:





           '''simple docstring'''

           A__   				=					[self.sep_token_id]
           A__   				=					[self.cls_token_id]
           if token_ids_a is None:
                  return len(cls + token_ids_a + sep) * [0]
           return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





    def  SCREAMING_SNAKE_CASE							(    self   :			int      ,				UpperCAmelCase__   :			str      ,				UpperCAmelCase__   :			Optional[str] = None)		->Tuple[str]:





           '''simple docstring'''

           A__   				=					self._tokenizer.model.save(UpperCAmelCase__      ,				name=UpperCAmelCase__)
           return tuple(UpperCAmelCase__)








from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCamelCase			:	Tuple							 =     logging.get_logger(__name__)

_lowerCamelCase			:	Tuple							 =     {
    """tiiuae/falcon-40b""": """https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json""",
    """tiiuae/falcon-7b""": """https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json""",
}


class        UpperCamelCase_					(				UpperCAmelCase__  ):

    '''simple docstring'''






    UpperCAmelCase__													=   '''falcon'''
    UpperCAmelCase__													=   ['''past_key_values''']


    def __init__(    self   :			Optional[Any]      ,				UpperCAmelCase__   :			str=65_024      ,				UpperCAmelCase__   :			List[Any]=4_544      ,				UpperCAmelCase__   :			Optional[Any]=32      ,				UpperCAmelCase__   :			Dict=71      ,				UpperCAmelCase__   :			Any=1e-5      ,				UpperCAmelCase__   :			Optional[int]=0.02      ,				UpperCAmelCase__   :			Optional[int]=True      ,				UpperCAmelCase__   :			Union[str, Any]=0.0      ,				UpperCAmelCase__   :			List[str]=0.0      ,				UpperCAmelCase__   :			str=None      ,				UpperCAmelCase__   :			Dict=False      ,				UpperCAmelCase__   :			List[str]=False      ,				UpperCAmelCase__   :			Optional[Any]=True      ,				UpperCAmelCase__   :			Optional[Any]=True      ,				UpperCAmelCase__   :			Optional[Any]=False      ,				UpperCAmelCase__   :			Dict=11      ,				UpperCAmelCase__   :			str=11      ,				**UpperCAmelCase__   :			Dict      ,				)		->Any:





           '''simple docstring'''

           A__   				=					vocab_size
           # Backward compatibility with n_embed kwarg
           A__   				=					kwargs.pop('''n_embed'''      ,				UpperCAmelCase__)
           A__   				=					hidden_size if n_embed is None else n_embed
           A__   				=					num_hidden_layers
           A__   				=					num_attention_heads
           A__   				=					layer_norm_epsilon
           A__   				=					initializer_range
           A__   				=					use_cache
           A__   				=					hidden_dropout
           A__   				=					attention_dropout

           A__   				=					bos_token_id
           A__   				=					eos_token_id
           A__   				=					num_attention_heads if num_kv_heads is None else num_kv_heads
           A__   				=					alibi
           A__   				=					new_decoder_architecture
           A__   				=					multi_query  # Ignored when new_decoder_architecture is True
           A__   				=					parallel_attn
           A__   				=					bias

           super().__init__(bos_token_id=UpperCAmelCase__      ,				eos_token_id=UpperCAmelCase__      ,				**UpperCAmelCase__)


    @property
    def  SCREAMING_SNAKE_CASE							(    self   :			str)		->int:





           '''simple docstring'''

           return self.hidden_size // self.num_attention_heads





    @property
    def  SCREAMING_SNAKE_CASE							(    self   :			str)		->Dict:





           '''simple docstring'''

           return not self.alibi


