





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase        =				logging.get_logger(__name__)

_UpperCAmelCase        =				'▁'

_UpperCAmelCase        =				{'vocab_file': 'sentencepiece.bpe.model'}

_UpperCAmelCase        =				{
    'vocab_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/blob/main/sentencepiece.bpe.model'
        ),
    }
}

_UpperCAmelCase        =				{
    'facebook/nllb-200-distilled-600M': 1_0_2_4,
}

# fmt: off
_UpperCAmelCase        =				['ace_Arab', 'ace_Latn', 'acm_Arab', 'acq_Arab', 'aeb_Arab', 'afr_Latn', 'ajp_Arab', 'aka_Latn', 'amh_Ethi', 'apc_Arab', 'arb_Arab', 'ars_Arab', 'ary_Arab', 'arz_Arab', 'asm_Beng', 'ast_Latn', 'awa_Deva', 'ayr_Latn', 'azb_Arab', 'azj_Latn', 'bak_Cyrl', 'bam_Latn', 'ban_Latn', 'bel_Cyrl', 'bem_Latn', 'ben_Beng', 'bho_Deva', 'bjn_Arab', 'bjn_Latn', 'bod_Tibt', 'bos_Latn', 'bug_Latn', 'bul_Cyrl', 'cat_Latn', 'ceb_Latn', 'ces_Latn', 'cjk_Latn', 'ckb_Arab', 'crh_Latn', 'cym_Latn', 'dan_Latn', 'deu_Latn', 'dik_Latn', 'dyu_Latn', 'dzo_Tibt', 'ell_Grek', 'eng_Latn', 'epo_Latn', 'est_Latn', 'eus_Latn', 'ewe_Latn', 'fao_Latn', 'pes_Arab', 'fij_Latn', 'fin_Latn', 'fon_Latn', 'fra_Latn', 'fur_Latn', 'fuv_Latn', 'gla_Latn', 'gle_Latn', 'glg_Latn', 'grn_Latn', 'guj_Gujr', 'hat_Latn', 'hau_Latn', 'heb_Hebr', 'hin_Deva', 'hne_Deva', 'hrv_Latn', 'hun_Latn', 'hye_Armn', 'ibo_Latn', 'ilo_Latn', 'ind_Latn', 'isl_Latn', 'ita_Latn', 'jav_Latn', 'jpn_Jpan', 'kab_Latn', 'kac_Latn', 'kam_Latn', 'kan_Knda', 'kas_Arab', 'kas_Deva', 'kat_Geor', 'knc_Arab', 'knc_Latn', 'kaz_Cyrl', 'kbp_Latn', 'kea_Latn', 'khm_Khmr', 'kik_Latn', 'kin_Latn', 'kir_Cyrl', 'kmb_Latn', 'kon_Latn', 'kor_Hang', 'kmr_Latn', 'lao_Laoo', 'lvs_Latn', 'lij_Latn', 'lim_Latn', 'lin_Latn', 'lit_Latn', 'lmo_Latn', 'ltg_Latn', 'ltz_Latn', 'lua_Latn', 'lug_Latn', 'luo_Latn', 'lus_Latn', 'mag_Deva', 'mai_Deva', 'mal_Mlym', 'mar_Deva', 'min_Latn', 'mkd_Cyrl', 'plt_Latn', 'mlt_Latn', 'mni_Beng', 'khk_Cyrl', 'mos_Latn', 'mri_Latn', 'zsm_Latn', 'mya_Mymr', 'nld_Latn', 'nno_Latn', 'nob_Latn', 'npi_Deva', 'nso_Latn', 'nus_Latn', 'nya_Latn', 'oci_Latn', 'gaz_Latn', 'ory_Orya', 'pag_Latn', 'pan_Guru', 'pap_Latn', 'pol_Latn', 'por_Latn', 'prs_Arab', 'pbt_Arab', 'quy_Latn', 'ron_Latn', 'run_Latn', 'rus_Cyrl', 'sag_Latn', 'san_Deva', 'sat_Beng', 'scn_Latn', 'shn_Mymr', 'sin_Sinh', 'slk_Latn', 'slv_Latn', 'smo_Latn', 'sna_Latn', 'snd_Arab', 'som_Latn', 'sot_Latn', 'spa_Latn', 'als_Latn', 'srd_Latn', 'srp_Cyrl', 'ssw_Latn', 'sun_Latn', 'swe_Latn', 'swh_Latn', 'szl_Latn', 'tam_Taml', 'tat_Cyrl', 'tel_Telu', 'tgk_Cyrl', 'tgl_Latn', 'tha_Thai', 'tir_Ethi', 'taq_Latn', 'taq_Tfng', 'tpi_Latn', 'tsn_Latn', 'tso_Latn', 'tuk_Latn', 'tum_Latn', 'tur_Latn', 'twi_Latn', 'tzm_Tfng', 'uig_Arab', 'ukr_Cyrl', 'umb_Latn', 'urd_Arab', 'uzn_Latn', 'vec_Latn', 'vie_Latn', 'war_Latn', 'wol_Latn', 'xho_Latn', 'ydd_Hebr', 'yor_Latn', 'yue_Hant', 'zho_Hans', 'zho_Hant', 'zul_Latn']



class 							_UpperCamelCase					(	lowerCAmelCase_     ):
       _UpperCamelCase				:					Dict         =   VOCAB_FILES_NAMES
       _UpperCamelCase				:					str         =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _UpperCamelCase				:					Tuple         =   PRETRAINED_VOCAB_FILES_MAP
       _UpperCamelCase				:					Union[str, Any]         =   ['''input_ids''', '''attention_mask''']

       _UpperCamelCase				:					List[int]         =   []
       _UpperCamelCase				:					List[int]         =   []

       def __init__(		self: Tuple					,   _SCREAMING_SNAKE_CASE: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: List[str]="<s>"					,   _SCREAMING_SNAKE_CASE: Optional[int]="</s>"					,   _SCREAMING_SNAKE_CASE: int="</s>"					,   _SCREAMING_SNAKE_CASE: Union[str, Any]="<s>"					,   _SCREAMING_SNAKE_CASE: Any="<unk>"					,   _SCREAMING_SNAKE_CASE: Union[str, Any]="<pad>"					,   _SCREAMING_SNAKE_CASE: int="<mask>"					,   _SCREAMING_SNAKE_CASE: Dict=None					,   _SCREAMING_SNAKE_CASE: Dict=None					,   _SCREAMING_SNAKE_CASE: int=None					,   _SCREAMING_SNAKE_CASE: Optional[Dict[str, Any]] = None					,   _SCREAMING_SNAKE_CASE: int=None					,   _SCREAMING_SNAKE_CASE: Tuple=False					,   **_SCREAMING_SNAKE_CASE: List[str]					,   )       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =  AddedToken(_SCREAMING_SNAKE_CASE					,   lstrip=_SCREAMING_SNAKE_CASE					,   rstrip=_SCREAMING_SNAKE_CASE		) if isinstance(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		) else mask_token

              UpperCamelCase_          =  {} if sp_model_kwargs is None else sp_model_kwargs
              UpperCamelCase_          =  legacy_behaviour

              super().__init__(
                  bos_token=_SCREAMING_SNAKE_CASE					,   eos_token=_SCREAMING_SNAKE_CASE					,   unk_token=_SCREAMING_SNAKE_CASE					,   sep_token=_SCREAMING_SNAKE_CASE					,   cls_token=_SCREAMING_SNAKE_CASE					,   pad_token=_SCREAMING_SNAKE_CASE					,   mask_token=_SCREAMING_SNAKE_CASE					,   tokenizer_file=_SCREAMING_SNAKE_CASE					,   src_lang=_SCREAMING_SNAKE_CASE					,   tgt_lang=_SCREAMING_SNAKE_CASE					,   additional_special_tokens=_SCREAMING_SNAKE_CASE					,   sp_model_kwargs=self.sp_model_kwargs					,   legacy_behaviour=_SCREAMING_SNAKE_CASE					,   **_SCREAMING_SNAKE_CASE					,   )

              UpperCamelCase_          =  spm.SentencePieceProcessor(**self.sp_model_kwargs		)
              self.sp_model.Load(str(_SCREAMING_SNAKE_CASE		)		)
              UpperCamelCase_          =  vocab_file
              # Original fairseq vocab and spm vocab must be "aligned":
              # Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
              # -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
              # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | 'an' | '▁n' | '▁m' | '▁t' | '▁k' | '▁a'
              # spm      | '<unk>' | '<s>'   | '</s>' | 'an'    | '▁n' | '▁m' | '▁t' | '▁k' | '▁a' | '▁s'

              # Mimic fairseq token-to-id alignment for the first 4 token
              UpperCamelCase_          =  {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

              # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
              UpperCamelCase_          =  1

              UpperCamelCase_          =  len(self.sp_model		)
              UpperCamelCase_          =  {
                  code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_SCREAMING_SNAKE_CASE		)
              }
              UpperCamelCase_          =  {v: k for k, v in self.lang_code_to_id.items()}
              UpperCamelCase_          =  len(self.sp_model		) + len(self.lang_code_to_id		) + self.fairseq_offset

              self.fairseq_tokens_to_ids.update(self.lang_code_to_id		)
              UpperCamelCase_          =  {v: k for k, v in self.fairseq_tokens_to_ids.items()}
              UpperCamelCase_          =  list(self.lang_code_to_id.keys()		)

              if additional_special_tokens is not None:
                     # Only add those special tokens if they are not already there.
                     self._additional_special_tokens.extend(
                         [t for t in additional_special_tokens if t not in self._additional_special_tokens]		)

              UpperCamelCase_          =  src_lang if src_lang is not None else "eng_Latn"
              UpperCamelCase_          =  self.lang_code_to_id[self._src_lang]
              UpperCamelCase_          =  tgt_lang
              self.set_src_lang_special_tokens(self._src_lang		)

       def __getstate__(		self: Any		)       -> Union[str, Any]:




              """simple docstring"""


              UpperCamelCase_          =  self.__dict__.copy()
              UpperCamelCase_          =  None
              UpperCamelCase_          =  self.sp_model.serialized_model_proto()
              return state

       def __setstate__(		self: List[Any]					,   _SCREAMING_SNAKE_CASE: Optional[Any]		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =  d

              # for backward compatibility
              if not hasattr(self					,   "sp_model_kwargs"		):
                     UpperCamelCase_          =  {}

              UpperCamelCase_          =  spm.SentencePieceProcessor(**self.sp_model_kwargs		)
              self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)

       @property
       def lowercase (		self: Union[str, Any]		)       -> Dict:




              """simple docstring"""


              return len(self.sp_model		) + len(self.lang_code_to_id		) + self.fairseq_offset + 1  # Plus 1 for the mask token

       @property
       def lowercase (		self: Union[str, Any]		)       -> str:




              """simple docstring"""


              return self._src_lang

       @src_lang.setter
       def lowercase (		self: Tuple					,   _SCREAMING_SNAKE_CASE: str		)       -> None:




              """simple docstring"""


              UpperCamelCase_          =  new_src_lang
              self.set_src_lang_special_tokens(self._src_lang		)

       def lowercase (		self: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: List[int]					,   _SCREAMING_SNAKE_CASE: Optional[List[int]] = None					,   _SCREAMING_SNAKE_CASE: bool = False		)       -> List[int]:




              """simple docstring"""


              if already_has_special_tokens:
                     return super().get_special_tokens_mask(
                         token_ids_a=_SCREAMING_SNAKE_CASE					,   token_ids_a=_SCREAMING_SNAKE_CASE					,   already_has_special_tokens=_SCREAMING_SNAKE_CASE		)

              UpperCamelCase_          =  [1] * len(self.prefix_tokens		)
              UpperCamelCase_          =  [1] * len(self.suffix_tokens		)
              if token_ids_a is None:
                     return prefix_ones + ([0] * len(_SCREAMING_SNAKE_CASE		)) + suffix_ones
              return prefix_ones + ([0] * len(_SCREAMING_SNAKE_CASE		)) + ([0] * len(_SCREAMING_SNAKE_CASE		)) + suffix_ones

       def lowercase (		self: Tuple					,   _SCREAMING_SNAKE_CASE: List[int]					,   _SCREAMING_SNAKE_CASE: Optional[List[int]] = None		)       -> List[int]:




              """simple docstring"""


              if token_ids_a is None:
                     return self.prefix_tokens + token_ids_a + self.suffix_tokens
              # We don't expect to process pairs, but leave the pair logic for API consistency
              return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

       def lowercase (		self: str					,   _SCREAMING_SNAKE_CASE: List[int]					,   _SCREAMING_SNAKE_CASE: Optional[List[int]] = None		)       -> List[int]:




              """simple docstring"""


              UpperCamelCase_          =  [self.sep_token_id]
              UpperCamelCase_          =  [self.cls_token_id]

              if token_ids_a is None:
                     return len(cls + token_ids_a + sep		) * [0]
              return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]

       def lowercase (		self: Tuple					,   _SCREAMING_SNAKE_CASE: Optional[int]					,   _SCREAMING_SNAKE_CASE: str					,   _SCREAMING_SNAKE_CASE: Optional[str]					,   _SCREAMING_SNAKE_CASE: Optional[str]					,   **_SCREAMING_SNAKE_CASE: Tuple		)       -> int:




              """simple docstring"""


              if src_lang is None or tgt_lang is None:
                     raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"		)
              UpperCamelCase_          =  src_lang
              UpperCamelCase_          =  self(_SCREAMING_SNAKE_CASE					,   add_special_tokens=_SCREAMING_SNAKE_CASE					,   return_tensors=_SCREAMING_SNAKE_CASE					,   **_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  self.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  tgt_lang_id
              return inputs

       def lowercase (		self: Tuple		)       -> Union[str, Any]:




              """simple docstring"""


              UpperCamelCase_          =  {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE		): i for i in range(self.vocab_size		)}
              vocab.update(self.added_tokens_encoder		)
              return vocab

       def lowercase (		self: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: str		)       -> List[str]:




              """simple docstring"""


              return self.sp_model.encode(_SCREAMING_SNAKE_CASE					,   out_type=_SCREAMING_SNAKE_CASE		)

       def lowercase (		self: Dict					,   _SCREAMING_SNAKE_CASE: str		)       -> Optional[int]:




              """simple docstring"""


              if token in self.fairseq_tokens_to_ids:
                     return self.fairseq_tokens_to_ids[token]
              UpperCamelCase_          =  self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE		)

              # Need to return unknown token if the SP model returned 0
              return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

       def lowercase (		self: int					,   _SCREAMING_SNAKE_CASE: Union[str, Any]		)       -> Any:




              """simple docstring"""


              if index in self.fairseq_ids_to_tokens:
                     return self.fairseq_ids_to_tokens[index]
              return self.sp_model.IdToPiece(index - self.fairseq_offset		)

       def lowercase (		self: int					,   _SCREAMING_SNAKE_CASE: Optional[int]		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =  "".join(_SCREAMING_SNAKE_CASE		).replace(_SCREAMING_SNAKE_CASE					,   " "		).strip()
              return out_string

       def lowercase (		self: str					,   _SCREAMING_SNAKE_CASE: str					,   _SCREAMING_SNAKE_CASE: Optional[str] = None		)       -> Tuple[str]:




              """simple docstring"""


              if not os.path.isdir(_SCREAMING_SNAKE_CASE		):
                     logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''		)
                     return
              UpperCamelCase_          =  os.path.join(
                  _SCREAMING_SNAKE_CASE					,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

              if os.path.abspath(self.vocab_file		) != os.path.abspath(_SCREAMING_SNAKE_CASE		) and os.path.isfile(self.vocab_file		):
                     copyfile(self.vocab_file					,   _SCREAMING_SNAKE_CASE		)
              elif not os.path.isfile(self.vocab_file		):
                     with open(_SCREAMING_SNAKE_CASE					,   "wb"		) as fi:
                            UpperCamelCase_          =  self.sp_model.serialized_model_proto()
                            fi.write(_SCREAMING_SNAKE_CASE		)

              return (out_vocab_file,)

       def lowercase (		self: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: List[str]					,   _SCREAMING_SNAKE_CASE: str = "eng_Latn"					,   _SCREAMING_SNAKE_CASE: Optional[List[str]] = None					,   _SCREAMING_SNAKE_CASE: str = "fra_Latn"					,   **_SCREAMING_SNAKE_CASE: List[str]					,   )       -> BatchEncoding:




              """simple docstring"""


              UpperCamelCase_          =  src_lang
              UpperCamelCase_          =  tgt_lang
              return super().prepare_seqaseq_batch(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE					,   **_SCREAMING_SNAKE_CASE		)

       def lowercase (		self: Any		)       -> Optional[int]:




              """simple docstring"""


              return self.set_src_lang_special_tokens(self.src_lang		)

       def lowercase (		self: Dict		)       -> Optional[int]:




              """simple docstring"""


              return self.set_tgt_lang_special_tokens(self.tgt_lang		)

       def lowercase (		self: List[str]					,   _SCREAMING_SNAKE_CASE: Any		)       -> None:




              """simple docstring"""


              UpperCamelCase_          =  self.lang_code_to_id[src_lang]
              if self.legacy_behaviour:
                     UpperCamelCase_          =  []
                     UpperCamelCase_          =  [self.eos_token_id, self.cur_lang_code]
              else:
                     UpperCamelCase_          =  [self.cur_lang_code]
                     UpperCamelCase_          =  [self.eos_token_id]



       def lowercase (		self: Any					,   _SCREAMING_SNAKE_CASE: str		)       -> None:




              """simple docstring"""


              UpperCamelCase_          =  self.lang_code_to_id[lang]
              if self.legacy_behaviour:
                     UpperCamelCase_          =  []
                     UpperCamelCase_          =  [self.eos_token_id, self.cur_lang_code]
              else:
                     UpperCamelCase_          =  [self.cur_lang_code]
                     UpperCamelCase_          =  [self.eos_token_id]








import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         FalconForCausalLM,
         FalconForQuestionAnswering,
         FalconForSequenceClassification,
         FalconForTokenClassification,
         FalconModel,
     )



class 							_UpperCamelCase					:

       def __init__(		self: Dict					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Optional[Any]=3					,   _SCREAMING_SNAKE_CASE: Any=7					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=True					,   _SCREAMING_SNAKE_CASE: Optional[int]=True					,   _SCREAMING_SNAKE_CASE: List[Any]=False					,   _SCREAMING_SNAKE_CASE: str=True					,   _SCREAMING_SNAKE_CASE: Any=99					,   _SCREAMING_SNAKE_CASE: List[str]=32					,   _SCREAMING_SNAKE_CASE: List[str]=5					,   _SCREAMING_SNAKE_CASE: Optional[int]=4					,   _SCREAMING_SNAKE_CASE: str=37					,   _SCREAMING_SNAKE_CASE: Optional[int]="gelu"					,   _SCREAMING_SNAKE_CASE: str=0.1					,   _SCREAMING_SNAKE_CASE: Any=0.1					,   _SCREAMING_SNAKE_CASE: Optional[Any]=512					,   _SCREAMING_SNAKE_CASE: Optional[Any]=16					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=2					,   _SCREAMING_SNAKE_CASE: Tuple=0.02					,   _SCREAMING_SNAKE_CASE: Optional[Any]=3					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=4					,   _SCREAMING_SNAKE_CASE: Tuple=None					,   )       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =  parent
              UpperCamelCase_          =  batch_size
              UpperCamelCase_          =  seq_length
              UpperCamelCase_          =  is_training
              UpperCamelCase_          =  use_input_mask
              UpperCamelCase_          =  use_token_type_ids
              UpperCamelCase_          =  use_labels
              UpperCamelCase_          =  vocab_size
              UpperCamelCase_          =  hidden_size
              UpperCamelCase_          =  num_hidden_layers
              UpperCamelCase_          =  num_attention_heads
              UpperCamelCase_          =  intermediate_size
              UpperCamelCase_          =  hidden_act
              UpperCamelCase_          =  hidden_dropout_prob
              UpperCamelCase_          =  attention_probs_dropout_prob
              UpperCamelCase_          =  max_position_embeddings
              UpperCamelCase_          =  type_vocab_size
              UpperCamelCase_          =  type_sequence_label_size
              UpperCamelCase_          =  initializer_range
              UpperCamelCase_          =  num_labels
              UpperCamelCase_          =  num_choices
              UpperCamelCase_          =  scope

       def lowercase (		self: Any		)       -> Any:




              """simple docstring"""


              UpperCamelCase_          =  ids_tensor([self.batch_size, self.seq_length]					,   self.vocab_size		)

              UpperCamelCase_          =  None
              if self.use_input_mask:
                     UpperCamelCase_          =  random_attention_mask([self.batch_size, self.seq_length]		)

              UpperCamelCase_          =  None

              UpperCamelCase_          =  None
              UpperCamelCase_          =  None
              UpperCamelCase_          =  None
              if self.use_labels:
                     UpperCamelCase_          =  ids_tensor([self.batch_size]					,   self.type_sequence_label_size		)
                     UpperCamelCase_          =  ids_tensor([self.batch_size, self.seq_length]					,   self.num_labels		)
                     UpperCamelCase_          =  ids_tensor([self.batch_size]					,   self.num_choices		)

              UpperCamelCase_          =  self.get_config()

              return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

       def lowercase (		self: Any		)       -> Union[str, Any]:




              """simple docstring"""


              return FalconConfig(
                  vocab_size=self.vocab_size					,   hidden_size=self.hidden_size					,   num_hidden_layers=self.num_hidden_layers					,   num_attention_heads=self.num_attention_heads					,   intermediate_size=self.intermediate_size					,   hidden_act=self.hidden_act					,   hidden_dropout_prob=self.hidden_dropout_prob					,   attention_probs_dropout_prob=self.attention_probs_dropout_prob					,   max_position_embeddings=self.max_position_embeddings					,   type_vocab_size=self.type_vocab_size					,   is_decoder=_SCREAMING_SNAKE_CASE					,   initializer_range=self.initializer_range					,   pad_token_id=1					,   new_decoder_architecture=_SCREAMING_SNAKE_CASE					,   )

       def lowercase (		self: str					,   _SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: List[str]					,   _SCREAMING_SNAKE_CASE: Dict					,   _SCREAMING_SNAKE_CASE: List[str]					,   _SCREAMING_SNAKE_CASE: int					,   _SCREAMING_SNAKE_CASE: Dict					,   _SCREAMING_SNAKE_CASE: Union[str, Any]		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =  FalconModel(config=_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE		)
              self.parent.assertEqual(result.last_hidden_state.shape					,   (self.batch_size, self.seq_length, self.hidden_size)		)

       def lowercase (		self: Optional[int]					,   _SCREAMING_SNAKE_CASE: Optional[int]					,   _SCREAMING_SNAKE_CASE: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: Optional[int]					,   _SCREAMING_SNAKE_CASE: Tuple					,   _SCREAMING_SNAKE_CASE: List[str]					,   _SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: Optional[int]					,   _SCREAMING_SNAKE_CASE: str					,   _SCREAMING_SNAKE_CASE: Tuple					,   )       -> str:




              """simple docstring"""


              UpperCamelCase_          =  True
              UpperCamelCase_          =  FalconModel(_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(
                  _SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   encoder_hidden_states=_SCREAMING_SNAKE_CASE					,   encoder_attention_mask=_SCREAMING_SNAKE_CASE					,   )
              UpperCamelCase_          =  model(
                  _SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   encoder_hidden_states=_SCREAMING_SNAKE_CASE					,   )
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE		)
              self.parent.assertEqual(result.last_hidden_state.shape					,   (self.batch_size, self.seq_length, self.hidden_size)		)

       def lowercase (		self: Optional[int]					,   _SCREAMING_SNAKE_CASE: Optional[int]					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: int					,   )       -> int:




              """simple docstring"""


              UpperCamelCase_          =  FalconForCausalLM(config=_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   labels=_SCREAMING_SNAKE_CASE		)
              self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.seq_length, self.vocab_size)		)

       def lowercase (		self: Optional[int]					,   _SCREAMING_SNAKE_CASE: Optional[int]					,   _SCREAMING_SNAKE_CASE: int					,   _SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: Optional[int]					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Tuple					,   _SCREAMING_SNAKE_CASE: Union[str, Any]					,   )       -> List[Any]:




              """simple docstring"""


              UpperCamelCase_          =  True
              UpperCamelCase_          =  True
              UpperCamelCase_          =  FalconForCausalLM(config=_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()

              # first forward pass
              UpperCamelCase_          =  model(
                  _SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   encoder_hidden_states=_SCREAMING_SNAKE_CASE					,   encoder_attention_mask=_SCREAMING_SNAKE_CASE					,   use_cache=_SCREAMING_SNAKE_CASE					,   )
              UpperCamelCase_          =  outputs.past_key_values

              # create hypothetical multiple next token and extent to next_input_ids
              UpperCamelCase_          =  ids_tensor((self.batch_size, 3)					,   config.vocab_size		)
              UpperCamelCase_          =  ids_tensor((self.batch_size, 3)					,   vocab_size=2		)

              # append to next input_ids and
              UpperCamelCase_          =  torch.cat([input_ids, next_tokens]					,   dim=-1		)
              UpperCamelCase_          =  torch.cat([input_mask, next_mask]					,   dim=-1		)

              UpperCamelCase_          =  model(
                  _SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   encoder_hidden_states=_SCREAMING_SNAKE_CASE					,   encoder_attention_mask=_SCREAMING_SNAKE_CASE					,   output_hidden_states=_SCREAMING_SNAKE_CASE					,   )["hidden_states"][0]
              UpperCamelCase_          =  model(
                  _SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   encoder_hidden_states=_SCREAMING_SNAKE_CASE					,   encoder_attention_mask=_SCREAMING_SNAKE_CASE					,   past_key_values=_SCREAMING_SNAKE_CASE					,   output_hidden_states=_SCREAMING_SNAKE_CASE					,   )["hidden_states"][0]

              # select random slice
              UpperCamelCase_          =  ids_tensor((1,)					,   output_from_past.shape[-1]		).item()
              UpperCamelCase_          =  output_from_no_past[:, -3:, random_slice_idx].detach()
              UpperCamelCase_          =  output_from_past[:, :, random_slice_idx].detach()

              self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]		)

              # test that outputs are equal for slice
              self.parent.assertTrue(torch.allclose(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE					,   atol=1e-3		)		)



       def lowercase (		self: Tuple		)       -> int:




              """simple docstring"""


              UpperCamelCase_          =  self.prepare_config_and_inputs()
              (
                  (
                  UpperCamelCase_
              )       , (
                  UpperCamelCase_
              )       , (
                  UpperCamelCase_
              )       , (
                  UpperCamelCase_
              )       , (
                  UpperCamelCase_
              )       , (
                  UpperCamelCase_
              )       , (
                  UpperCamelCase_
              )       , 
              )          =  config_and_inputs
              UpperCamelCase_          =  {"input_ids": input_ids, "attention_mask": input_mask}
              return config, inputs_dict



@require_torch
class 							_UpperCamelCase					(	lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						unittest.TestCase     ):
       _UpperCamelCase				:					Optional[Any]         =   (
           (
               FalconModel,
               FalconForCausalLM,
               FalconForSequenceClassification,
               FalconForTokenClassification,
               FalconForQuestionAnswering,
           )
           if is_torch_available()
           else ()
       )
       _UpperCamelCase				:					Union[str, Any]         =   (FalconForCausalLM,) if is_torch_available() else ()
       _UpperCamelCase				:					Optional[int]         =   (
           {
               '''feature-extraction''': FalconModel,
               '''text-classification''': FalconForSequenceClassification,
               '''text-generation''': FalconForCausalLM,
               '''question-answering''': FalconForQuestionAnswering,
               '''token-classification''': FalconForTokenClassification,
               '''zero-shot''': FalconForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       _UpperCamelCase				:					Any         =   False
       _UpperCamelCase				:					int         =   False

       def lowercase (		self: str		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =  FalconModelTester(self		)
              UpperCamelCase_          =  ConfigTester(self					,   config_class=_SCREAMING_SNAKE_CASE					,   hidden_size=37		)

       def lowercase (		self: Optional[Any]		)       -> Optional[int]:




              """simple docstring"""


              self.config_tester.run_common_tests()

       def lowercase (		self: str		)       -> Any:




              """simple docstring"""


              UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE		)

       def lowercase (		self: List[Any]		)       -> Dict:




              """simple docstring"""


              UpperCamelCase_       , *UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs()
              for alibi in [True, False]:
                     UpperCamelCase_          =  alibi
                     self.model_tester.create_and_check_model(_SCREAMING_SNAKE_CASE					,   *_SCREAMING_SNAKE_CASE		)

       def lowercase (		self: Any		)       -> str:




              """simple docstring"""


              UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()
              UpperCamelCase_          =  3
              UpperCamelCase_          =  input_dict["input_ids"]
              UpperCamelCase_          =  input_ids.ne(1		).to(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  ids_tensor([self.model_tester.batch_size]					,   self.model_tester.type_sequence_label_size		)
              UpperCamelCase_          =  FalconForSequenceClassification(_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   labels=_SCREAMING_SNAKE_CASE		)
              self.assertEqual(result.logits.shape					,   (self.model_tester.batch_size, self.model_tester.num_labels)		)

       def lowercase (		self: Dict		)       -> List[str]:




              """simple docstring"""


              UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()
              UpperCamelCase_          =  3
              UpperCamelCase_          =  "single_label_classification"
              UpperCamelCase_          =  input_dict["input_ids"]
              UpperCamelCase_          =  input_ids.ne(1		).to(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  ids_tensor([self.model_tester.batch_size]					,   self.model_tester.type_sequence_label_size		)
              UpperCamelCase_          =  FalconForSequenceClassification(_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   labels=_SCREAMING_SNAKE_CASE		)
              self.assertEqual(result.logits.shape					,   (self.model_tester.batch_size, self.model_tester.num_labels)		)

       def lowercase (		self: List[str]		)       -> int:




              """simple docstring"""


              UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()
              UpperCamelCase_          =  input_dict["input_ids"]
              UpperCamelCase_          =  FalconForCausalLM(_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   use_cache=_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  input_ids.shape[0]
              UpperCamelCase_          =  model._convert_to_rw_cache(result.past_key_values		)
              UpperCamelCase_          =  model._convert_cache_to_standard_format(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		)
              for layer in range(len(_SCREAMING_SNAKE_CASE		)		):
                     for tensor_idx in range(2		):
                            self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3		)
                            self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4		)
                            self.assertTrue(
                                torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]		)		)

       def lowercase (		self: Union[str, Any]		)       -> Any:




              """simple docstring"""


              UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()
              UpperCamelCase_          =  3
              UpperCamelCase_          =  "multi_label_classification"
              UpperCamelCase_          =  input_dict["input_ids"]
              UpperCamelCase_          =  input_ids.ne(1		).to(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  ids_tensor(
                  [self.model_tester.batch_size, config.num_labels]					,   self.model_tester.type_sequence_label_size		).to(torch.float		)
              UpperCamelCase_          =  FalconForSequenceClassification(_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   labels=_SCREAMING_SNAKE_CASE		)
              self.assertEqual(result.logits.shape					,   (self.model_tester.batch_size, self.model_tester.num_labels)		)



       def lowercase (		self: List[Any]		)       -> List[Any]:




              """simple docstring"""


              for model_class in self.all_generative_model_classes:
                     UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()

                     # If it doesn't support cache, pass the test
                     if not hasattr(_SCREAMING_SNAKE_CASE					,   "use_cache"		):
                            return

                     UpperCamelCase_          =  model_class(_SCREAMING_SNAKE_CASE		).to(_SCREAMING_SNAKE_CASE		)
                     if "use_cache" not in inputs:
                            UpperCamelCase_          =  True
                     UpperCamelCase_          =  model(**_SCREAMING_SNAKE_CASE		)

                     # If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
                     if "past_key_values" not in outputs:
                            return

                     UpperCamelCase_          =  (
                         getattr(_SCREAMING_SNAKE_CASE					,   "decoder_layers"					,   _SCREAMING_SNAKE_CASE		)
                         or getattr(_SCREAMING_SNAKE_CASE					,   "num_decoder_layers"					,   _SCREAMING_SNAKE_CASE		)
                         or config.num_hidden_layers
                     )
                     UpperCamelCase_          =  getattr(_SCREAMING_SNAKE_CASE					,   "num_kv_heads"					,   config.num_attention_heads		)
                     UpperCamelCase_          =  getattr(_SCREAMING_SNAKE_CASE					,   "d_model"					,   config.hidden_size		)
                     UpperCamelCase_          =  embed_dim // num_attention_heads

                     UpperCamelCase_          =  outputs["past_key_values"]
                     self.assertEqual(len(_SCREAMING_SNAKE_CASE		)					,   _SCREAMING_SNAKE_CASE		)

                     UpperCamelCase_       , UpperCamelCase_          =  inputs["input_ids"].shape
                     for i in range(_SCREAMING_SNAKE_CASE		):
                            if config.new_decoder_architecture:
                                   UpperCamelCase_          =  config.num_attention_heads
                            elif config.multi_query:
                                   UpperCamelCase_          =  1
                            self.assertEqual(len(past_kv[0]		)					,   2		)  # K V for the decoder = 2
                            self.assertEqual(
                                past_kv[i][0].shape					,   (batch_size, num_attention_heads, seq_length, per_head_embed_dim)		)
                            self.assertEqual(
                                past_kv[i][1].shape					,   (batch_size, num_attention_heads, seq_length, per_head_embed_dim)		)



@require_torch
class 							_UpperCamelCase					(	unittest.TestCase     ):

       @slow
       def lowercase (		self: Tuple		)       -> int:




              """simple docstring"""


              UpperCamelCase_          =  AutoTokenizer.from_pretrained("Rocketknight1/falcon-rw-1b"		)
              UpperCamelCase_          =  FalconForCausalLM.from_pretrained("Rocketknight1/falcon-rw-1b"		)
              model.eval()
              model.to(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  tokenizer("My favorite food is"					,   return_tensors="pt"		).to(_SCREAMING_SNAKE_CASE		)

              UpperCamelCase_          =  (
                  "My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday."
              )

              UpperCamelCase_          =  model.generate(**_SCREAMING_SNAKE_CASE					,   do_sample=_SCREAMING_SNAKE_CASE					,   max_new_tokens=19		)
              UpperCamelCase_          =  tokenizer.batch_decode(_SCREAMING_SNAKE_CASE		)[0]

              self.assertEqual(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		)

       @slow
       def lowercase (		self: Dict		)       -> Dict:




              """simple docstring"""


              for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
                     UpperCamelCase_          =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		)
                     UpperCamelCase_          =  FalconForCausalLM.from_pretrained(_SCREAMING_SNAKE_CASE		)
                     model.eval()
                     model.to(_SCREAMING_SNAKE_CASE		)
                     UpperCamelCase_          =  tokenizer("My favorite food is"					,   return_tensors="pt"		).to(_SCREAMING_SNAKE_CASE		)

                     # We just test that these run without errors - the models are randomly initialized
                     # and so the actual text outputs will be garbage
                     model.generate(**_SCREAMING_SNAKE_CASE					,   do_sample=_SCREAMING_SNAKE_CASE					,   max_new_tokens=4		)
                     model.generate(**_SCREAMING_SNAKE_CASE					,   do_sample=_SCREAMING_SNAKE_CASE					,   max_new_tokens=4		)
                     model.generate(**_SCREAMING_SNAKE_CASE					,   num_beams=2					,   max_new_tokens=4		)



       @slow
       def lowercase (		self: Union[str, Any]		)       -> Any:




              """simple docstring"""


              with torch.no_grad():
                     for repo in [
                         "Rocketknight1/falcon-rw-1b",
                         "Rocketknight1/tiny-random-falcon-7b",
                         "Rocketknight1/tiny-random-falcon-40b",
                     ]:
                            UpperCamelCase_          =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		)
                            UpperCamelCase_          =  FalconForCausalLM.from_pretrained(_SCREAMING_SNAKE_CASE		)
                            model.eval()
                            model.to(device=_SCREAMING_SNAKE_CASE		)
                            UpperCamelCase_          =  tokenizer("My favorite food is"					,   return_tensors="pt"		).to(_SCREAMING_SNAKE_CASE		)

                            # Test results are the same with and without cache
                            UpperCamelCase_          =  model.generate(**_SCREAMING_SNAKE_CASE					,   do_sample=_SCREAMING_SNAKE_CASE					,   max_new_tokens=20					,   use_cache=_SCREAMING_SNAKE_CASE		)
                            UpperCamelCase_          =  model.generate(**_SCREAMING_SNAKE_CASE					,   do_sample=_SCREAMING_SNAKE_CASE					,   max_new_tokens=20					,   use_cache=_SCREAMING_SNAKE_CASE		)
                            self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0		)

