





'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


_SCREAMING_SNAKE_CASE    =			logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE    =			{'''vocab_file''': '''vocab.txt'''}

_SCREAMING_SNAKE_CASE    =			{
    '''vocab_file''': {
        '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt''',
        '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt''',
        '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt''',
        '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt''',
    }
}

_SCREAMING_SNAKE_CASE    =			{
    '''tau/splinter-base''': 5_1_2,
    '''tau/splinter-base-qass''': 5_1_2,
    '''tau/splinter-large''': 5_1_2,
    '''tau/splinter-large-qass''': 5_1_2,
}

_SCREAMING_SNAKE_CASE    =			{
    '''tau/splinter-base''': {'''do_lower_case''': False},
    '''tau/splinter-base-qass''': {'''do_lower_case''': False},
    '''tau/splinter-large''': {'''do_lower_case''': False},
    '''tau/splinter-large-qass''': {'''do_lower_case''': False},
}






class 							__lowercase				(					lowerCAmelCase__  ):







	'''simple docstring'''

	a			: Optional[Any]      =				VOCAB_FILES_NAMES
	a			: Dict      =				PRETRAINED_VOCAB_FILES_MAP
	a			: Optional[Any]      =				PRETRAINED_INIT_CONFIGURATION
	a			: Tuple      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a			: Tuple      =				SplinterTokenizer


	def __init__(self   ,_lowerCamelCase=None   ,_lowerCamelCase=None   ,_lowerCamelCase=True   ,_lowerCamelCase="[UNK]"   ,_lowerCamelCase="[SEP]"   ,_lowerCamelCase="[PAD]"   ,_lowerCamelCase="[CLS]"   ,_lowerCamelCase="[MASK]"   ,_lowerCamelCase="[QUESTION]"   ,_lowerCamelCase=True   ,_lowerCamelCase=None   ,**_lowerCamelCase   ,)							->							List[Any]:







				'''simple docstring'''





				super().__init__(
				    _lowerCamelCase   ,tokenizer_file=_lowerCamelCase   ,do_lower_case=_lowerCamelCase   ,unk_token=_lowerCamelCase   ,sep_token=_lowerCamelCase   ,pad_token=_lowerCamelCase   ,cls_token=_lowerCamelCase   ,mask_token=_lowerCamelCase   ,tokenize_chinese_chars=_lowerCamelCase   ,strip_accents=_lowerCamelCase   ,additional_special_tokens=(question_token,)   ,**_lowerCamelCase   ,)

				__lowercase   =							json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
				if (
				    pre_tok_state.get('''lowercase'''   ,_lowerCamelCase       ) != do_lower_case
				    or pre_tok_state.get('''strip_accents'''   ,_lowerCamelCase       ) != strip_accents
				):
							__lowercase   =							getattr(_lowerCamelCase   ,pre_tok_state.pop('''type'''       )       )
							__lowercase   =							do_lower_case
							__lowercase   =							strip_accents
							__lowercase   =							pre_tok_class(**_lowerCamelCase       )

				__lowercase   =							do_lower_case


	@property
	def   _UpperCAmelCase   (self       )							->							int:







				'''simple docstring'''





				return self.convert_tokens_to_ids(self.question_token       )


	def   _UpperCAmelCase   (self   ,_lowerCamelCase   ,_lowerCamelCase = None       )							->							List[int]:







				'''simple docstring'''





				if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

				__lowercase   =							[self.cls_token_id]
				__lowercase   =							[self.sep_token_id]
				__lowercase   =							[self.question_token_id] + [self.convert_tokens_to_ids('''.'''       )]
				if self.padding_side == "right":
							# Input is question-then-context
							return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
				else:
							# Input is context-then-question
							return cls + token_ids_a + sep + token_ids_a + question_suffix + sep


	def   _UpperCAmelCase   (self   ,_lowerCamelCase   ,_lowerCamelCase = None       )							->							List[int]:







				'''simple docstring'''





				__lowercase   =							[self.sep_token_id]
				__lowercase   =							[self.cls_token_id]
				__lowercase   =							[self.question_token_id] + [self.convert_tokens_to_ids('''.'''       )]
				if token_ids_a is None:
							return len(cls + token_ids_a + sep       ) * [0]

				if self.padding_side == "right":
							# Input is question-then-context
							return len(cls + token_ids_a + question_suffix + sep       ) * [0] + len(token_ids_a + sep       ) * [1]
				else:
							# Input is context-then-question
							return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + question_suffix + sep       ) * [1]


	def   _UpperCAmelCase   (self   ,_lowerCamelCase   ,_lowerCamelCase = None       )							->							Tuple[str]:







				'''simple docstring'''





				__lowercase   =							self._tokenizer.model.save(_lowerCamelCase   ,name=_lowerCamelCase       )
				return tuple(_lowerCamelCase       )











'''simple docstring'''



from __future__ import annotations

import unittest

from transformers import CTRLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers.models.ctrl.modeling_tf_ctrl import (
         TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFCTRLForSequenceClassification,
         TFCTRLLMHeadModel,
         TFCTRLModel,
     )





class 			lowerCAmelCase      (       __lowerCAmelCase):


       def __init__(      self			,     __SCREAMING_SNAKE_CASE			,     )					->     Any:



            '''simple docstring'''




            __snake_case			    =     parent
            __snake_case			    =     13
            __snake_case			    =     7
            __snake_case			    =     True
            __snake_case			    =     True
            __snake_case			    =     True
            __snake_case			    =     True
            __snake_case			    =     True
            __snake_case			    =     99
            __snake_case			    =     32
            __snake_case			    =     2
            __snake_case			    =     4
            __snake_case			    =     37
            __snake_case			    =     '''gelu'''
            __snake_case			    =     0.1
            __snake_case			    =     0.1
            __snake_case			    =     512
            __snake_case			    =     16
            __snake_case			    =     2
            __snake_case			    =     0.02
            __snake_case			    =     3
            __snake_case			    =     4
            __snake_case			    =     None
            __snake_case			    =     self.vocab_size - 1


       def 					lowerCAmelCase					(      self     )					->     Any:



            '''simple docstring'''




            __snake_case			    =     ids_tensor([self.batch_size, self.seq_length]			,     self.vocab_size     )

            __snake_case			    =     None
            if self.use_input_mask:
                 __snake_case			    =     random_attention_mask([self.batch_size, self.seq_length]     )

            __snake_case			    =     None
            if self.use_token_type_ids:
                 __snake_case			    =     ids_tensor([self.batch_size, self.seq_length]			,     self.type_vocab_size     )

            __snake_case			    =     None
            if self.use_mc_token_ids:
                 __snake_case			    =     ids_tensor([self.batch_size, self.num_choices]			,     self.seq_length     )

            __snake_case			    =     None
            __snake_case			    =     None
            __snake_case			    =     None
            if self.use_labels:
                 __snake_case			    =     ids_tensor([self.batch_size]			,     self.type_sequence_label_size     )
                 __snake_case			    =     ids_tensor([self.batch_size, self.seq_length]			,     self.num_labels     )
                 __snake_case			    =     ids_tensor([self.batch_size]			,     self.num_choices     )

            __snake_case			    =     CTRLConfig(
                vocab_size=self.vocab_size			,     n_embd=self.hidden_size			,     n_layer=self.num_hidden_layers			,     n_head=self.num_attention_heads			,     n_positions=self.max_position_embeddings			,     pad_token_id=self.pad_token_id			,     )

            __snake_case			    =     ids_tensor([self.num_hidden_layers, self.num_attention_heads]			,     2     )

            return (
                config,
                input_ids,
                input_mask,
                head_mask,
                token_type_ids,
                mc_token_ids,
                sequence_labels,
                token_labels,
                choice_labels,
            )


       def 					lowerCAmelCase					(      self			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     *__SCREAMING_SNAKE_CASE     )					->     List[Any]:



            '''simple docstring'''




            __snake_case			    =     TFCTRLModel(config=__SCREAMING_SNAKE_CASE     )
            __snake_case			    =     {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
            __snake_case			    =     model(__SCREAMING_SNAKE_CASE     )

            __snake_case			    =     [input_ids, None, input_mask]  # None is the input for 'past'
            __snake_case			    =     model(__SCREAMING_SNAKE_CASE     )

            __snake_case			    =     model(__SCREAMING_SNAKE_CASE     )

            self.parent.assertEqual(result.last_hidden_state.shape			,     (self.batch_size, self.seq_length, self.hidden_size)     )


       def 					lowerCAmelCase					(      self			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     *__SCREAMING_SNAKE_CASE     )					->     int:



            '''simple docstring'''




            __snake_case			    =     TFCTRLLMHeadModel(config=__SCREAMING_SNAKE_CASE     )
            __snake_case			    =     {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
            __snake_case			    =     model(__SCREAMING_SNAKE_CASE     )
            self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.seq_length, self.vocab_size)     )


       def 					lowerCAmelCase					(      self			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     *__SCREAMING_SNAKE_CASE     )					->     List[str]:



            '''simple docstring'''




            __snake_case			    =     self.num_labels
            __snake_case			    =     ids_tensor([self.batch_size]			,     self.type_sequence_label_size     )
            __snake_case			    =     {
                '''input_ids''': input_ids,
                '''token_type_ids''': token_type_ids,
                '''labels''': sequence_labels,
            }
            __snake_case			    =     TFCTRLForSequenceClassification(__SCREAMING_SNAKE_CASE     )
            __snake_case			    =     model(__SCREAMING_SNAKE_CASE     )
            self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.num_labels)     )




       def 					lowerCAmelCase					(      self     )					->     Optional[Any]:



            '''simple docstring'''




            __snake_case			    =     self.prepare_config_and_inputs()

            (
                (
                __snake_case
            )    ,			(
                __snake_case
            )    ,			(
                __snake_case
            )    ,			(
                __snake_case
            )    ,			(
                __snake_case
            )    ,			(
                __snake_case
            )    ,			(
                __snake_case
            )    ,			(
                __snake_case
            )    ,			(
                __snake_case
            )    ,			
            )			    =     config_and_inputs

            __snake_case			    =     {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
            return config, inputs_dict





@require_tf
class 			lowerCAmelCase      (       __lowerCAmelCase ,      __lowerCAmelCase ,      unittest.TestCase):
       __lowercase     :					str       				=	(TFCTRLModel, TFCTRLLMHeadModel, TFCTRLForSequenceClassification) if is_tf_available() else ()
       __lowercase     :					Tuple       				=	(TFCTRLLMHeadModel,) if is_tf_available() else ()
       __lowercase     :					Optional[Any]       				=	(
           {
               '''feature-extraction''': TFCTRLModel,
               '''text-classification''': TFCTRLForSequenceClassification,
               '''text-generation''': TFCTRLLMHeadModel,
               '''zero-shot''': TFCTRLForSequenceClassification,
           }
           if is_tf_available()
           else {}
       )
       __lowercase     :					List[Any]       				=	False
       __lowercase     :					Optional[int]       				=	False


       def 					lowerCAmelCase					(      self			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     )					->     Dict:



            '''simple docstring'''




            if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
                 # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
                 # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
                 # config could not be created.
                 return True

            return False


       def 					lowerCAmelCase					(      self     )					->     Optional[int]:



            '''simple docstring'''




            __snake_case			    =     TFCTRLModelTester(self     )
            __snake_case			    =     ConfigTester(self			,     config_class=__SCREAMING_SNAKE_CASE			,     n_embd=37     )


       def 					lowerCAmelCase					(      self     )					->     Dict:



            '''simple docstring'''




            self.config_tester.run_common_tests()


       def 					lowerCAmelCase					(      self     )					->     Union[str, Any]:



            '''simple docstring'''




            __snake_case			    =     self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_ctrl_model(*__SCREAMING_SNAKE_CASE     )


       def 					lowerCAmelCase					(      self     )					->     Optional[int]:



            '''simple docstring'''




            __snake_case			    =     self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_ctrl_lm_head(*__SCREAMING_SNAKE_CASE     )


       def 					lowerCAmelCase					(      self     )					->     List[str]:



            '''simple docstring'''




            __snake_case			    =     self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_ctrl_for_sequence_classification(*__SCREAMING_SNAKE_CASE     )


       def 					lowerCAmelCase					(      self     )					->     Optional[int]:



            '''simple docstring'''




            __snake_case    ,			__snake_case			    =     self.model_tester.prepare_config_and_inputs_for_common()
            __snake_case			    =     [TFCTRLLMHeadModel]
            __snake_case			    =     [TFCTRLForSequenceClassification]

            for model_class in self.all_model_classes:
                 __snake_case			    =     model_class(__SCREAMING_SNAKE_CASE     )
                 model.build()  # may be needed for the get_bias() call below
                 assert isinstance(model.get_input_embeddings()			,     tf.keras.layers.Layer     )

                 if model_class in list_lm_models:
                      __snake_case			    =     model.get_output_embeddings()
                      assert isinstance(__SCREAMING_SNAKE_CASE			,     tf.keras.layers.Layer     )
                      __snake_case			    =     model.get_bias()
                      assert isinstance(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     )
                      for k, v in name.items():
                           assert isinstance(__SCREAMING_SNAKE_CASE			,     tf.Variable     )
                 elif model_class in list_other_models_with_output_ebd:
                      __snake_case			    =     model.get_output_embeddings()
                      assert isinstance(__SCREAMING_SNAKE_CASE			,     tf.keras.layers.Layer     )
                      __snake_case			    =     model.get_bias()
                      assert name is None
                 else:
                      __snake_case			    =     model.get_output_embeddings()
                      assert x is None
                      __snake_case			    =     model.get_bias()
                      assert name is None




       @slow
       def 					lowerCAmelCase					(      self     )					->     Union[str, Any]:



            '''simple docstring'''




            for model_name in TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 __snake_case			    =     TFCTRLModel.from_pretrained(__SCREAMING_SNAKE_CASE     )
                 self.assertIsNotNone(__SCREAMING_SNAKE_CASE     )





@require_tf
class 			lowerCAmelCase      (       unittest.TestCase):


       @slow
       def 					lowerCAmelCase					(      self     )					->     Tuple:



            '''simple docstring'''




            __snake_case			    =     TFCTRLLMHeadModel.from_pretrained('''ctrl'''     )
            __snake_case			    =     tf.convert_to_tensor([[1_1859, 0, 1611, 8]]			,     dtype=tf.intaa     )  # Legal the president is
            __snake_case			    =     [
                1_1859,
                0,
                1611,
                8,
                5,
                150,
                2_6449,
                2,
                19,
                348,
                469,
                3,
                2595,
                48,
                2_0740,
                24_6533,
                24_6533,
                19,
                30,
                5,
            ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

            __snake_case			    =     model.generate(__SCREAMING_SNAKE_CASE			,     do_sample=__SCREAMING_SNAKE_CASE     )
            self.assertListEqual(output_ids[0].numpy().tolist()			,     __SCREAMING_SNAKE_CASE     )






