





'''simple docstring'''

import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin
class     __lowerCAmelCase					(   __magic_name__						,						unittest.TestCase			):






       """simple docstring"""
       _snake_case     :					Optional[Any]						    =   DebertaTokenizer
       _snake_case     :					Union[str, Any]						    =   True
       _snake_case     :					List[str]						    =   DebertaTokenizerFast





       def snake_case__						(							self       :		List[str]					)   ->					Optional[Any]:

            '''simple docstring'''




            super().setUp()

            # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
            _UpperCamelCase       =	[
                '''l''',
                '''o''',
                '''w''',
                '''e''',
                '''r''',
                '''s''',
                '''t''',
                '''i''',
                '''d''',
                '''n''',
                '''\u0120''',
                '''\u0120l''',
                '''\u0120n''',
                '''\u0120lo''',
                '''\u0120low''',
                '''er''',
                '''\u0120lowest''',
                '''\u0120newer''',
                '''\u0120wider''',
                '''[UNK]''',
            ]
            _UpperCamelCase       =	dict(zip(lowerCAmelCase__	,			range(len(lowerCAmelCase__					)					)					)					)
            _UpperCamelCase       =	['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
            _UpperCamelCase       =	{'''unk_token''': '''[UNK]'''}

            _UpperCamelCase       =	os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES['''vocab_file''']					)
            _UpperCamelCase       =	os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES['''merges_file''']					)
            with open(self.vocab_file	,			'''w'''	,			encoding='''utf-8'''					) as fp:
                 fp.write(json.dumps(lowerCAmelCase__					) + '''\n'''					)
            with open(self.merges_file	,			'''w'''	,			encoding='''utf-8'''					) as fp:
                 fp.write('''\n'''.join(lowerCAmelCase__					)					)





       def snake_case__						(							self       :		Optional[int]	,			**lowerCAmelCase__       :		List[Any]					)   ->					Tuple:

            '''simple docstring'''




            kwargs.update(self.special_tokens_map					)
            return self.tokenizer_class.from_pretrained(self.tmpdirname	,			**lowerCAmelCase__					)





       def snake_case__						(							self       :		str	,			lowerCAmelCase__       :		str					)   ->					List[Any]:

            '''simple docstring'''




            _UpperCamelCase       =	'''lower newer'''
            _UpperCamelCase       =	'''lower newer'''
            return input_text, output_text





       def snake_case__						(							self       :		str					)   ->					List[str]:

            '''simple docstring'''




            _UpperCamelCase       =	self.get_tokenizer()
            _UpperCamelCase       =	'''lower newer'''
            _UpperCamelCase       =	['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
            _UpperCamelCase       =	tokenizer.tokenize(lowerCAmelCase__					)
            self.assertListEqual(lowerCAmelCase__	,			lowerCAmelCase__					)

            _UpperCamelCase       =	tokens + [tokenizer.unk_token]
            _UpperCamelCase       =	[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
            self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__					)	,			lowerCAmelCase__					)





       def snake_case__						(							self       :		int					)   ->					Optional[int]:

            '''simple docstring'''




            _UpperCamelCase       =	self.get_tokenizer()
            _UpperCamelCase       =	tokenizer('''Hello'''	,			'''World'''					)
            _UpperCamelCase       =	[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
            self.assertListEqual(tokd['''token_type_ids''']	,			lowerCAmelCase__					)





       @slow
       def snake_case__						(							self       :		List[Any]					)   ->					int:

            '''simple docstring'''




            _UpperCamelCase       =	self.tokenizer_class.from_pretrained('''microsoft/deberta-base'''					)

            _UpperCamelCase       =	tokenizer.encode('''sequence builders'''	,			add_special_tokens=lowerCAmelCase__					)
            _UpperCamelCase       =	tokenizer.encode('''multi-sequence build'''	,			add_special_tokens=lowerCAmelCase__					)

            _UpperCamelCase       =	tokenizer.encode(
                '''sequence builders'''	,			add_special_tokens=lowerCAmelCase__	,			add_prefix_space=lowerCAmelCase__					)
            _UpperCamelCase       =	tokenizer.encode(
                '''sequence builders'''	,			'''multi-sequence build'''	,			add_special_tokens=lowerCAmelCase__	,			add_prefix_space=lowerCAmelCase__					)

            _UpperCamelCase       =	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__					)
            _UpperCamelCase       =	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__	,			lowerCAmelCase__					)

            assert encoded_sentence == encoded_text_from_decode
            assert encoded_pair == encoded_pair_from_decode







       @slow
       def snake_case__						(							self       :		int					)   ->					List[Any]:

            '''simple docstring'''




            _UpperCamelCase       =	[self.tokenizer_class]
            if self.test_rust_tokenizer:
                 tokenizer_classes.append(self.rust_tokenizer_class					)

            for tokenizer_class in tokenizer_classes:
                 _UpperCamelCase       =	tokenizer_class.from_pretrained('''microsoft/deberta-base'''					)

                 _UpperCamelCase       =	[
                     '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
                     '''ALBERT incorporates two parameter reduction techniques''',
                     '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
                     ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
                     ''' vocabulary embedding.''',
                 ]

                 _UpperCamelCase       =	tokenizer(lowerCAmelCase__	,			padding=lowerCAmelCase__					)
                 _UpperCamelCase       =	[tokenizer.decode(lowerCAmelCase__	,			skip_special_tokens=lowerCAmelCase__					) for seq in encoding['''input_ids''']]

                 # fmt: off
                 _UpperCamelCase       =	{
                     '''input_ids''': [
                         [1, 2118, 11126, 565, 35, 83, 25191, 163, 18854, 13, 12156, 12, 16101, 25376, 13807, 9, 22205, 27893, 1635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                         [1, 2118, 11126, 565, 24536, 80, 43797, 4878, 7373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                         [1, 133, 78, 65, 16, 10, 3724, 1538, 33183, 11303, 43797, 1938, 4, 870, 24165, 29105, 5, 739, 32644, 33183, 11303, 36173, 88, 80, 650, 7821, 45940, 6, 52, 2559, 5, 1836, 9, 5, 7397, 13171, 31, 5, 1836, 9, 32644, 33183, 11303, 4, 2]
                     ],
                     '''token_type_ids''': [
                         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                     ],
                     '''attention_mask''': [
                         [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                         [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                         [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
                     ]
                 }
                 # fmt: on

                 _UpperCamelCase       =	[
                     '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
                     '''ALBERT incorporates two parameter reduction techniques''',
                     '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
                     ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
                     ''' vocabulary embedding.''',
                 ]

                 self.assertDictEqual(encoding.data	,			lowerCAmelCase__					)

                 for expected, decoded in zip(lowerCAmelCase__	,			lowerCAmelCase__					):
                      self.assertEqual(lowerCAmelCase__	,			lowerCAmelCase__					)










'''simple docstring'''

import unittest

import numpy as np

from transformers import RobertaConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
   from transformers.models.roberta.modeling_flax_roberta import (
       FlaxRobertaForCausalLM,
       FlaxRobertaForMaskedLM,
       FlaxRobertaForMultipleChoice,
       FlaxRobertaForQuestionAnswering,
       FlaxRobertaForSequenceClassification,
       FlaxRobertaForTokenClassification,
       FlaxRobertaModel,
   )
class     __lowerCAmelCase					(   unittest.TestCase			):






       """simple docstring"""





       def __init__(							self       :		Tuple	,			lowerCAmelCase__       :		Tuple	,			lowerCAmelCase__       :		Union[str, Any]=13	,			lowerCAmelCase__       :		Union[str, Any]=7	,			lowerCAmelCase__       :		str=True	,			lowerCAmelCase__       :		Tuple=True	,			lowerCAmelCase__       :		Dict=True	,			lowerCAmelCase__       :		Dict=True	,			lowerCAmelCase__       :		int=99	,			lowerCAmelCase__       :		str=32	,			lowerCAmelCase__       :		str=5	,			lowerCAmelCase__       :		str=4	,			lowerCAmelCase__       :		str=37	,			lowerCAmelCase__       :		int="gelu"	,			lowerCAmelCase__       :		Optional[Any]=0.1	,			lowerCAmelCase__       :		int=0.1	,			lowerCAmelCase__       :		Optional[int]=512	,			lowerCAmelCase__       :		Dict=16	,			lowerCAmelCase__       :		List[Any]=2	,			lowerCAmelCase__       :		Any=0.02	,			lowerCAmelCase__       :		Union[str, Any]=4	,			)   ->					Dict:

            '''simple docstring'''




            _UpperCamelCase       =	parent
            _UpperCamelCase       =	batch_size
            _UpperCamelCase       =	seq_length
            _UpperCamelCase       =	is_training
            _UpperCamelCase       =	use_attention_mask
            _UpperCamelCase       =	use_token_type_ids
            _UpperCamelCase       =	use_labels
            _UpperCamelCase       =	vocab_size
            _UpperCamelCase       =	hidden_size
            _UpperCamelCase       =	num_hidden_layers
            _UpperCamelCase       =	num_attention_heads
            _UpperCamelCase       =	intermediate_size
            _UpperCamelCase       =	hidden_act
            _UpperCamelCase       =	hidden_dropout_prob
            _UpperCamelCase       =	attention_probs_dropout_prob
            _UpperCamelCase       =	max_position_embeddings
            _UpperCamelCase       =	type_vocab_size
            _UpperCamelCase       =	type_sequence_label_size
            _UpperCamelCase       =	initializer_range
            _UpperCamelCase       =	num_choices





       def snake_case__						(							self       :		Optional[int]					)   ->					Optional[Any]:

            '''simple docstring'''




            _UpperCamelCase       =	ids_tensor([self.batch_size, self.seq_length]	,			self.vocab_size					)

            _UpperCamelCase       =	None
            if self.use_attention_mask:
                 _UpperCamelCase       =	random_attention_mask([self.batch_size, self.seq_length]					)

            _UpperCamelCase       =	None
            if self.use_token_type_ids:
                 _UpperCamelCase       =	ids_tensor([self.batch_size, self.seq_length]	,			self.type_vocab_size					)

            _UpperCamelCase       =	RobertaConfig(
                vocab_size=self.vocab_size	,			hidden_size=self.hidden_size	,			num_hidden_layers=self.num_hidden_layers	,			num_attention_heads=self.num_attention_heads	,			intermediate_size=self.intermediate_size	,			hidden_act=self.hidden_act	,			hidden_dropout_prob=self.hidden_dropout_prob	,			attention_probs_dropout_prob=self.attention_probs_dropout_prob	,			max_position_embeddings=self.max_position_embeddings	,			type_vocab_size=self.type_vocab_size	,			is_decoder=lowerCAmelCase__	,			initializer_range=self.initializer_range	,			)

            return config, input_ids, token_type_ids, attention_mask





       def snake_case__						(							self       :		Optional[int]					)   ->					Any:

            '''simple docstring'''




            _UpperCamelCase       =	self.prepare_config_and_inputs()
            _UpperCamelCase   ,    _UpperCamelCase   ,    _UpperCamelCase   ,    _UpperCamelCase       =	config_and_inputs
            _UpperCamelCase       =	{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': attention_mask}
            return config, inputs_dict







       def snake_case__						(							self       :		List[str]					)   ->					Any:

            '''simple docstring'''




            _UpperCamelCase       =	self.prepare_config_and_inputs()
            _UpperCamelCase   ,    _UpperCamelCase   ,    _UpperCamelCase   ,    _UpperCamelCase       =	config_and_inputs

            _UpperCamelCase       =	True
            _UpperCamelCase       =	floats_tensor([self.batch_size, self.seq_length, self.hidden_size]					)
            _UpperCamelCase       =	ids_tensor([self.batch_size, self.seq_length]	,			vocab_size=2					)

            return (
                config,
                input_ids,
                token_type_ids,
                encoder_hidden_states,
                encoder_attention_mask,
            )

@require_flax
class     __lowerCAmelCase					(   __magic_name__						,						unittest.TestCase			):






       """simple docstring"""
       _snake_case     :					Optional[int]						    =   True

       _snake_case     :					Optional[Any]						    =   (
           (
               FlaxRobertaModel,
               FlaxRobertaForCausalLM,
               FlaxRobertaForMaskedLM,
               FlaxRobertaForSequenceClassification,
               FlaxRobertaForTokenClassification,
               FlaxRobertaForMultipleChoice,
               FlaxRobertaForQuestionAnswering,
           )
           if is_flax_available()
           else ()
       )





       def snake_case__						(							self       :		Dict					)   ->					List[Any]:

            '''simple docstring'''




            _UpperCamelCase       =	FlaxRobertaModelTester(self					)







       @slow
       def snake_case__						(							self       :		List[str]					)   ->					Union[str, Any]:

            '''simple docstring'''




            for model_class_name in self.all_model_classes:
                 _UpperCamelCase       =	model_class_name.from_pretrained('''roberta-base'''	,			from_pt=lowerCAmelCase__					)
                 _UpperCamelCase       =	model(np.ones((1, 1)					)					)
                 self.assertIsNotNone(lowerCAmelCase__					)



