





"""simple docstring"""






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import numpy as np
		import tensorflow as tf

		from transformers import (
		    TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    FlaubertConfig,
		    TFFlaubertForMultipleChoice,
		    TFFlaubertForQuestionAnsweringSimple,
		    TFFlaubertForSequenceClassification,
		    TFFlaubertForTokenClassification,
		    TFFlaubertModel,
		    TFFlaubertWithLMHeadModel,
		)




class 					__lowercase      :







					'''simple docstring'''




					def __init__(    self						,      _UpperCAmelCase						,      ):
									__a			:    Optional[Any]      							=	parent
									__a			:    Tuple      							=	13
									__a			:    List[str]      							=	7
									__a			:    Dict      							=	True
									__a			:    Optional[int]      							=	True
									__a			:    List[str]      							=	True
									__a			:    Union[str, Any]      							=	True
									__a			:    List[Any]      							=	True
									__a			:    Tuple      							=	False
									__a			:    Optional[Any]      							=	False
									__a			:    Optional[int]      							=	False
									__a			:    List[Any]      							=	2
									__a			:    Dict      							=	99
									__a			:    int      							=	0
									__a			:    Any      							=	32
									__a			:    str      							=	2
									__a			:    Tuple      							=	4
									__a			:    List[Any]      							=	0.1
									__a			:    Optional[int]      							=	0.1
									__a			:    Union[str, Any]      							=	512
									__a			:    Optional[Any]      							=	16
									__a			:    Optional[int]      							=	2
									__a			:    str      							=	0.0_2
									__a			:    str      							=	3
									__a			:    Any      							=	4
									__a			:    Union[str, Any]      							=	'''last'''
									__a			:    Tuple      							=	True
									__a			:    Optional[int]      							=	None
									__a			:    Union[str, Any]      							=	0







					def 			_lowerCamelCase		(    self  ):
									__a			:    Optional[Any]      							=	ids_tensor([self.batch_size, self.seq_length]						,      self.vocab_size  )
									__a			:    Union[str, Any]      							=	random_attention_mask([self.batch_size, self.seq_length]						,      dtype=tf.floataa  )

									__a			:    Any      							=	None
									if self.use_input_lengths:
													__a			:    Optional[Any]      							=	(
													    ids_tensor([self.batch_size]						,      vocab_size=2  ) + self.seq_length - 2
													)  # small variation of seq_length

									__a			:    str      							=	None
									if self.use_token_type_ids:
													__a			:    str      							=	ids_tensor([self.batch_size, self.seq_length]						,      self.n_langs  )

									__a			:    Tuple      							=	None
									__a			:    Optional[Any]      							=	None
									__a			:    Any      							=	None
									if self.use_labels:
													__a			:    Optional[Any]      							=	ids_tensor([self.batch_size]						,      self.type_sequence_label_size  )
													__a			:    List[Any]      							=	ids_tensor([self.batch_size, self.seq_length]						,      self.num_labels  )
													__a			:    int      							=	ids_tensor([self.batch_size]						,      2						,      dtype=tf.floataa  )
													__a			:    Any      							=	ids_tensor([self.batch_size]						,      self.num_choices  )

									__a			:    List[Any]      							=	FlaubertConfig(
									    vocab_size=self.vocab_size						,      n_special=self.n_special						,      emb_dim=self.hidden_size						,      n_layers=self.num_hidden_layers						,      n_heads=self.num_attention_heads						,      dropout=self.hidden_dropout_prob						,      attention_dropout=self.attention_probs_dropout_prob						,      gelu_activation=self.gelu_activation						,      sinusoidal_embeddings=self.sinusoidal_embeddings						,      asm=self.asm						,      causal=self.causal						,      n_langs=self.n_langs						,      max_position_embeddings=self.max_position_embeddings						,      initializer_range=self.initializer_range						,      summary_type=self.summary_type						,      use_proj=self.use_proj						,      bos_token_id=self.bos_token_id						,      )

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_lengths,
									    sequence_labels,
									    token_labels,
									    is_impossible_labels,
									    choice_labels,
									    input_mask,
									)







					def 			_lowerCamelCase		(    self						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      ):
									__a			:    Optional[Any]      							=	TFFlaubertModel(config=A_  )
									__a			:    Optional[Any]      							=	{'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
									__a			:    Tuple      							=	model(A_  )

									__a			:    List[Any]      							=	[input_ids, input_mask]
									__a			:    Optional[int]      							=	model(A_  )
									self.parent.assertEqual(result.last_hidden_state.shape						,      (self.batch_size, self.seq_length, self.hidden_size)  )







					def 			_lowerCamelCase		(    self						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      ):
									__a			:    Tuple      							=	TFFlaubertWithLMHeadModel(A_  )

									__a			:    int      							=	{'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
									__a			:    Tuple      							=	model(A_  )

									self.parent.assertEqual(result.logits.shape						,      (self.batch_size, self.seq_length, self.vocab_size)  )







					def 			_lowerCamelCase		(    self						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      ):
									__a			:    Any      							=	TFFlaubertForQuestionAnsweringSimple(A_  )

									__a			:    Optional[Any]      							=	{'''input_ids''': input_ids, '''lengths''': input_lengths}

									__a			:    Union[str, Any]      							=	model(A_  )

									self.parent.assertEqual(result.start_logits.shape						,      (self.batch_size, self.seq_length)  )
									self.parent.assertEqual(result.end_logits.shape						,      (self.batch_size, self.seq_length)  )







					def 			_lowerCamelCase		(    self						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      ):
									__a			:    List[Any]      							=	TFFlaubertForSequenceClassification(A_  )

									__a			:    Union[str, Any]      							=	{'''input_ids''': input_ids, '''lengths''': input_lengths}

									__a			:    List[Any]      							=	model(A_  )

									self.parent.assertEqual(result.logits.shape						,      (self.batch_size, self.type_sequence_label_size)  )







					def 			_lowerCamelCase		(    self						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      ):
									__a			:    List[str]      							=	self.num_labels
									__a			:    Union[str, Any]      							=	TFFlaubertForTokenClassification(config=A_  )
									__a			:    List[Any]      							=	{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
									__a			:    Dict      							=	model(A_  )
									self.parent.assertEqual(result.logits.shape						,      (self.batch_size, self.seq_length, self.num_labels)  )







					def 			_lowerCamelCase		(    self						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      ):
									__a			:    Dict      							=	self.num_choices
									__a			:    Optional[int]      							=	TFFlaubertForMultipleChoice(config=A_  )
									__a			:    List[Any]      							=	tf.tile(tf.expand_dims(A_						,      1  )						,      (1, self.num_choices, 1)  )
									__a			:    Tuple      							=	tf.tile(tf.expand_dims(A_						,      1  )						,      (1, self.num_choices, 1)  )
									__a			:    int      							=	tf.tile(tf.expand_dims(A_						,      1  )						,      (1, self.num_choices, 1)  )
									__a			:    Any      							=	{
									    '''input_ids''': multiple_choice_inputs_ids,
									    '''attention_mask''': multiple_choice_input_mask,
									    '''token_type_ids''': multiple_choice_token_type_ids,
									}
									__a			:    Union[str, Any]      							=	model(A_  )
									self.parent.assertEqual(result.logits.shape						,      (self.batch_size, self.num_choices)  )







					def 			_lowerCamelCase		(    self  ):
									__a			:    Union[str, Any]      							=	self.prepare_config_and_inputs()
									(
									    (
									    __a		
									)							,    (
									    __a		
									)							,    (
									    __a		
									)							,    (
									    __a		
									)							,    (
									    __a		
									)							,    (
									    __a		
									)							,    (
									    __a		
									)							,    (
									    __a		
									)							,    (
									    __a		
									)							,    		
									)			:    List[Any]      							=	config_and_inputs
									__a			:    Union[str, Any]      							=	{
									    '''input_ids''': input_ids,
									    '''token_type_ids''': token_type_ids,
									    '''langs''': token_type_ids,
									    '''lengths''': input_lengths,
									}
									return config, inputs_dict






@require_tf
class 					__lowercase      (      __SCREAMING_SNAKE_CASE ,   __SCREAMING_SNAKE_CASE ,   unittest.TestCase				):







					'''simple docstring'''




					__lowerCAmelCase	=	(
					    (
					        TFFlaubertModel,
					        TFFlaubertWithLMHeadModel,
					        TFFlaubertForSequenceClassification,
					        TFFlaubertForQuestionAnsweringSimple,
					        TFFlaubertForTokenClassification,
					        TFFlaubertForMultipleChoice,
					    )
					    if is_tf_available()
					    else ()
					)
					__lowerCAmelCase	=	(
					    (TFFlaubertWithLMHeadModel,) if is_tf_available() else ()
					)  # TODO (PVP): Check other models whether language generation is also applicable
					__lowerCAmelCase	=	(
					    {
					        """feature-extraction""": TFFlaubertModel,
					        """fill-mask""": TFFlaubertWithLMHeadModel,
					        """question-answering""": TFFlaubertForQuestionAnsweringSimple,
					        """text-classification""": TFFlaubertForSequenceClassification,
					        """token-classification""": TFFlaubertForTokenClassification,
					        """zero-shot""": TFFlaubertForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					__lowerCAmelCase	=	False
					__lowerCAmelCase	=	False







					def 			_lowerCamelCase		(    self						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase						,      _UpperCAmelCase  ):

									if (
									    pipeline_test_casse_name == "QAPipelineTests"
									    and tokenizer_name is not None
									    and not tokenizer_name.endswith('''Fast'''  )
									):
													# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
													# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
													# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
													return True

									return False







					def 			_lowerCamelCase		(    self  ):
									__a			:    Optional[int]      							=	TFFlaubertModelTester(self  )
									__a			:    str      							=	ConfigTester(self						,      config_class=A_						,      emb_dim=37  )







					def 			_lowerCamelCase		(    self  ):
									self.config_tester.run_common_tests()







					def 			_lowerCamelCase		(    self  ):
									__a			:    Optional[Any]      							=	self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_model(*A_  )







					def 			_lowerCamelCase		(    self  ):
									__a			:    Union[str, Any]      							=	self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_lm_head(*A_  )







					def 			_lowerCamelCase		(    self  ):
									__a			:    List[str]      							=	self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_qa(*A_  )







					def 			_lowerCamelCase		(    self  ):
									__a			:    str      							=	self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_sequence_classif(*A_  )







					def 			_lowerCamelCase		(    self  ):
									__a			:    str      							=	self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_for_token_classification(*A_  )







					def 			_lowerCamelCase		(    self  ):
									__a			:    str      							=	self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_for_multiple_choice(*A_  )







					@slow
					def 			_lowerCamelCase		(    self  ):

									for model_name in TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													__a			:    Union[str, Any]      							=	TFFlaubertModel.from_pretrained(A_  )
													self.assertIsNotNone(A_  )






@require_tf
@require_sentencepiece
@require_tokenizers
class 					__lowercase      (      unittest.TestCase				):







					'''simple docstring'''




					@slow
					def 			_lowerCamelCase		(    self  ):
									__a			:    str      							=	TFFlaubertModel.from_pretrained('''jplu/tf-flaubert-small-cased'''  )

									__a			:    Union[str, Any]      							=	tf.convert_to_tensor(
									    [[0, 158, 735, 2592, 1424, 6727, 82, 1]]						,      dtype=tf.intaa						,      )  # "J'aime flaubert !"

									__a			:    Any      							=	model(A_  )[0]
									__a			:    str      							=	tf.TensorShape((1, 8, 512)  )
									self.assertEqual(output.shape						,      A_  )
									# compare the actual values for a slice.
									__a			:    Any      							=	tf.convert_to_tensor(
									    [
									        [
									            [-1.8_7_6_8_7_7_3, -1.5_6_6_5_5_5, 0.2_7_0_7_2_4_1_8],
									            [-1.6_9_2_0_0_3_8, -0.5_8_7_3_5_0_5, 1.9_3_2_9_5_9_9],
									            [-2.9_5_6_3_9_8_5, -1.6_9_9_3_8_3_5, 1.7_9_7_2_0_5_2],
									        ]
									    ]						,      dtype=tf.floataa						,      )

									self.assertTrue(np.allclose(output[:, :3, :3].numpy()						,      expected_slice.numpy()						,      atol=1e-4  )  )






import json
import os
import unittest

from transformers.models.biogpt.tokenization_biogpt import VOCAB_FILES_NAMES, BioGptTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin



class 					__snake_case	(  __SCREAMING_SNAKE_CASE			,     unittest.TestCase    ):


    '''simple docstring'''







    lowerCamelCase__    :		Dict				=	BioGptTokenizer
    lowerCamelCase__    :		List[Any]				=	False



    def  lowercase_					(       self     ):





        '''simple docstring'''

        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        SCREAMING_SNAKE_CASE__											=			[
            '''l''',
            '''o''',
            '''w''',
            '''e''',
            '''r''',
            '''s''',
            '''t''',
            '''i''',
            '''d''',
            '''n''',
            '''w</w>''',
            '''r</w>''',
            '''t</w>''',
            '''lo''',
            '''low''',
            '''er</w>''',
            '''low</w>''',
            '''lowest</w>''',
            '''newer</w>''',
            '''wider</w>''',
            '''<unk>''',
        ]
        SCREAMING_SNAKE_CASE__											=			dict(zip(A_   ,				range(len(A_     )     )     )     )
        SCREAMING_SNAKE_CASE__											=			['''l o 123''', '''lo w 1456''', '''e r</w> 1789''', '''''']

        SCREAMING_SNAKE_CASE__											=			os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''vocab_file''']     )
        SCREAMING_SNAKE_CASE__											=			os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''merges_file''']     )
        with open(self.vocab_file   ,				'''w'''     ) as fp:
            fp.write(json.dumps(A_     )     )
        with open(self.merges_file   ,				'''w'''     ) as fp:
            fp.write('''\n'''.join(A_     )     )



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			'''lower newer'''
        SCREAMING_SNAKE_CASE__											=			'''lower newer'''
        return input_text, output_text



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			BioGptTokenizer(self.vocab_file   ,				self.merges_file     )

        SCREAMING_SNAKE_CASE__											=			'''lower'''
        SCREAMING_SNAKE_CASE__											=			['''low''', '''er</w>''']
        SCREAMING_SNAKE_CASE__											=			tokenizer.tokenize(A_     )
        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			tokens + ['''<unk>''']
        SCREAMING_SNAKE_CASE__											=			[14, 15, 20]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(A_     )   ,				A_     )




    @slow
    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			BioGptTokenizer.from_pretrained('''microsoft/biogpt'''     )

        SCREAMING_SNAKE_CASE__											=			tokenizer.encode('''sequence builders'''   ,				add_special_tokens=A_     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.encode('''multi-sequence build'''   ,				add_special_tokens=A_     )

        SCREAMING_SNAKE_CASE__											=			tokenizer.build_inputs_with_special_tokens(A_     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.build_inputs_with_special_tokens(A_   ,				A_     )

        self.assertTrue(encoded_sentence == [2] + text     )
        self.assertTrue(encoded_pair == [2] + text + [2] + text_a     )
