
"""simple docstring"""

import json
import os
import unittest

from transformers import BatchEncoding, MvpTokenizer, MvpTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin, filter_roberta_detectors
@require_tokenizers
class   _UpperCAmelCase				(		SCREAMING_SNAKE_CASE_  ,      unittest.TestCase				):
      __SCREAMING_SNAKE_CASE :				Optional[int]				   =						MvpTokenizer
      __SCREAMING_SNAKE_CASE :				List[Any]				   =						MvpTokenizerFast
      __SCREAMING_SNAKE_CASE :				Any				   =						True
      __SCREAMING_SNAKE_CASE :				Any				   =						filter_roberta_detectors




      def      a_   (				self				)    ->       Dict:
          super().setUp()
          UpperCAmelCase          =       [
              'l',
              'o',
              'w',
              'e',
              'r',
              's',
              't',
              'i',
              'd',
              'n',
              '\u0120',
              '\u0120l',
              '\u0120n',
              '\u0120lo',
              '\u0120low',
              'er',
              '\u0120lowest',
              '\u0120newer',
              '\u0120wider',
              '<unk>',
          ]
          UpperCAmelCase          =       dict(zip(lowercase_ ,							range(len(lowercase_				)				)				)				)
          UpperCAmelCase          =       ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
          UpperCAmelCase          =       {'unk_token': '<unk>'}

          UpperCAmelCase          =       os.path.join(self.tmpdirname ,							VOCAB_FILES_NAMES['vocab_file']				)
          UpperCAmelCase          =       os.path.join(self.tmpdirname ,							VOCAB_FILES_NAMES['merges_file']				)
          with open(self.vocab_file ,							'w' ,							encoding='utf-8'				) as fp:
              fp.write(json.dumps(lowercase_				) + '\n'				)
          with open(self.merges_file ,							'w' ,							encoding='utf-8'				) as fp:
              fp.write('\n'.join(lowercase_				)				)




      def      a_   (				self ,							**lowercase_				)    ->       Dict:
          kwargs.update(self.special_tokens_map				)
          return self.tokenizer_class.from_pretrained(self.tmpdirname ,							**lowercase_				)




      def      a_   (				self ,							**lowercase_				)    ->       Union[str, Any]:
          kwargs.update(self.special_tokens_map				)
          return self.rust_tokenizer_class.from_pretrained(self.tmpdirname ,							**lowercase_				)




      def      a_   (				self ,							lowercase_				)    ->       Union[str, Any]:
          return "lower newer", "lower newer"




      @cached_property
      def      a_   (				self				)    ->       str:
          return MvpTokenizer.from_pretrained('RUCAIBox/mvp'				)




      @cached_property
      def      a_   (				self				)    ->       Optional[int]:
          return MvpTokenizerFast.from_pretrained('RUCAIBox/mvp'				)




      @require_torch
      def      a_   (				self				)    ->       Any:
          UpperCAmelCase          =       ['A long paragraph for summarization.', 'Another paragraph for summarization.']
          UpperCAmelCase          =       [0, 2_5_0, 2_5_1, 1_7_8_1_8, 1_3, 3_9_1_8_6, 1_9_3_8, 4, 2]

          for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
              UpperCAmelCase          =       tokenizer(lowercase_ ,							max_length=len(lowercase_				) ,							padding=lowercase_ ,							return_tensors='pt'				)
              self.assertIsInstance(lowercase_ ,							lowercase_				)

              self.assertEqual((2, 9) ,							batch.input_ids.shape				)
              self.assertEqual((2, 9) ,							batch.attention_mask.shape				)
              UpperCAmelCase          =       batch.input_ids.tolist()[0]
              self.assertListEqual(lowercase_ ,							lowercase_				)
              # Test that special tokens are reset




      @require_torch
      def      a_   (				self				)    ->       Union[str, Any]:
          UpperCAmelCase          =       ['A long paragraph for summarization.', 'Another paragraph for summarization.']
          for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
              UpperCAmelCase          =       tokenizer(lowercase_ ,							padding=lowercase_ ,							return_tensors='pt'				)
              # check if input_ids are returned and no labels
              self.assertIn('input_ids' ,							lowercase_				)
              self.assertIn('attention_mask' ,							lowercase_				)
              self.assertNotIn('labels' ,							lowercase_				)
              self.assertNotIn('decoder_attention_mask' ,							lowercase_				)




      @require_torch
      def      a_   (				self				)    ->       Optional[Any]:
          UpperCAmelCase          =       [
              'Summary of the text.',
              'Another summary.',
          ]
          for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
              UpperCAmelCase          =       tokenizer(text_target=lowercase_ ,							max_length=3_2 ,							padding='max_length' ,							return_tensors='pt'				)
              self.assertEqual(3_2 ,							targets['input_ids'].shape[1]				)




      @require_torch
      def      a_   (				self				)    ->       Optional[Any]:
          for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
              UpperCAmelCase          =       tokenizer(
                  ['I am a small frog' * 1_0_2_4, 'I am a small frog'] ,							padding=lowercase_ ,							truncation=lowercase_ ,							return_tensors='pt'				)
              self.assertIsInstance(lowercase_ ,							lowercase_				)
              self.assertEqual(batch.input_ids.shape ,							(2, 1_0_2_4)				)




      @require_torch
      def      a_   (				self				)    ->       Tuple:
          UpperCAmelCase          =       ['A long paragraph for summarization.']
          UpperCAmelCase          =       [
              'Summary of the text.',
          ]
          for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
              UpperCAmelCase          =       tokenizer(lowercase_ ,							text_target=lowercase_ ,							return_tensors='pt'				)
              UpperCAmelCase          =       inputs['input_ids']
              UpperCAmelCase          =       inputs['labels']
              self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()				)
              self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()				)
              self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()				)
              self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()				)




      def      a_   (				self				)    ->       Optional[int]:
          pass




      def      a_   (				self				)    ->       Optional[int]:
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"				):
                  UpperCAmelCase          =       self.rust_tokenizer_class.from_pretrained(lowercase_ ,							**lowercase_				)
                  UpperCAmelCase          =       self.tokenizer_class.from_pretrained(lowercase_ ,							**lowercase_				)
                  UpperCAmelCase          =       'A, <mask> AllenNLP sentence.'
                  UpperCAmelCase          =       tokenizer_r.encode_plus(lowercase_ ,							add_special_tokens=lowercase_ ,							return_token_type_ids=lowercase_				)
                  UpperCAmelCase          =       tokenizer_p.encode_plus(lowercase_ ,							add_special_tokens=lowercase_ ,							return_token_type_ids=lowercase_				)

                  # token_type_ids should put 0 everywhere
                  self.assertEqual(sum(tokens_r['token_type_ids']				) ,							sum(tokens_p['token_type_ids']				)				)

                  # attention_mask should put 1 everywhere, so sum over length should be 1
                  self.assertEqual(
                      sum(tokens_r['attention_mask']				) / len(tokens_r['attention_mask']				) ,							sum(tokens_p['attention_mask']				) / len(tokens_p['attention_mask']				) ,							)

                  UpperCAmelCase          =       tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']				)
                  UpperCAmelCase          =       tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']				)

                  # Rust correctly handles the space before the mask while python doesnt
                  self.assertSequenceEqual(tokens_p['input_ids'] ,							[0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]				)
                  self.assertSequenceEqual(tokens_r['input_ids'] ,							[0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]				)

                  self.assertSequenceEqual(
                      lowercase_ ,							['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']				)
                  self.assertSequenceEqual(
                      lowercase_ ,							['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']				)





"""simple docstring"""

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_            =    logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_            =    {
    '''tiiuae/falcon-40b''': '''https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json''',
    '''tiiuae/falcon-7b''': '''https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json''',
}
class   _UpperCAmelCase				(		SCREAMING_SNAKE_CASE_				):
      __SCREAMING_SNAKE_CASE :				Optional[Any]				   =						"falcon"
      __SCREAMING_SNAKE_CASE :				List[Any]				   =						["past_key_values"]




      def __init__(				self ,							lowercase_=6_5_0_2_4 ,							lowercase_=4_5_4_4 ,							lowercase_=3_2 ,							lowercase_=7_1 ,							lowercase_=1E-5 ,							lowercase_=0.0_2 ,							lowercase_=True ,							lowercase_=0.0 ,							lowercase_=0.0 ,							lowercase_=None ,							lowercase_=False ,							lowercase_=False ,							lowercase_=True ,							lowercase_=True ,							lowercase_=False ,							lowercase_=1_1 ,							lowercase_=1_1 ,							**lowercase_ ,							)    ->       Any:
          UpperCAmelCase          =       vocab_size
          # Backward compatibility with n_embed kwarg
          UpperCAmelCase          =       kwargs.pop('n_embed' ,							lowercase_				)
          UpperCAmelCase          =       hidden_size if n_embed is None else n_embed
          UpperCAmelCase          =       num_hidden_layers
          UpperCAmelCase          =       num_attention_heads
          UpperCAmelCase          =       layer_norm_epsilon
          UpperCAmelCase          =       initializer_range
          UpperCAmelCase          =       use_cache
          UpperCAmelCase          =       hidden_dropout
          UpperCAmelCase          =       attention_dropout

          UpperCAmelCase          =       bos_token_id
          UpperCAmelCase          =       eos_token_id
          UpperCAmelCase          =       num_attention_heads if num_kv_heads is None else num_kv_heads
          UpperCAmelCase          =       alibi
          UpperCAmelCase          =       new_decoder_architecture
          UpperCAmelCase          =       multi_query  # Ignored when new_decoder_architecture is True
          UpperCAmelCase          =       parallel_attn
          UpperCAmelCase          =       bias

          super().__init__(bos_token_id=lowercase_ ,							eos_token_id=lowercase_ ,							**lowercase_				)




      @property
      def      a_   (				self				)    ->       List[Any]:
          return self.hidden_size // self.num_attention_heads




      @property
      def      a_   (				self				)    ->       List[str]:
          return not self.alibi



