
'''simple docstring'''







import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin


class 		__UpperCAmelCase   (      _lowerCamelCase		,     unittest.TestCase		):
							__lowercase   	= DebertaTokenizer
							__lowercase   	= True
							__lowercase   	= DebertaTokenizerFast


							def 				lowerCamelCase       ( self						):





								"""simple docstring"""






								super().setUp()

								# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
								_snake_case     		=	[
								    'l',
								    'o',
								    'w',
								    'e',
								    'r',
								    's',
								    't',
								    'i',
								    'd',
								    'n',
								    '\u0120',
								    '\u0120l',
								    '\u0120n',
								    '\u0120lo',
								    '\u0120low',
								    'er',
								    '\u0120lowest',
								    '\u0120newer',
								    '\u0120wider',
								    '[UNK]',
								]
								_snake_case     		=	dict(zip(lowerCAmelCase_					,     range(len(lowerCAmelCase_						)						)						)						)
								_snake_case     		=	['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
								_snake_case     		=	{'unk_token': '[UNK]'}

								_snake_case     		=	os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['vocab_file']						)
								_snake_case     		=	os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['merges_file']						)
								with open(self.vocab_file					,     'w'					,     encoding='utf-8'						) as fp:
									fp.write(json.dumps(lowerCAmelCase_						) + '\n'						)
								with open(self.merges_file					,     'w'					,     encoding='utf-8'						) as fp:
									fp.write('\n'.join(lowerCAmelCase_						)						)


							def 				lowerCamelCase       ( self					,     **lowerCAmelCase_						):





								"""simple docstring"""






								kwargs.update(self.special_tokens_map						)
								return self.tokenizer_class.from_pretrained(self.tmpdirname					,     **lowerCAmelCase_						)


							def 				lowerCamelCase       ( self					,     lowerCAmelCase_						):





								"""simple docstring"""






								_snake_case     		=	'lower newer'
								_snake_case     		=	'lower newer'
								return input_text, output_text


							def 				lowerCamelCase       ( self						):





								"""simple docstring"""






								_snake_case     		=	self.get_tokenizer()
								_snake_case     		=	'lower newer'
								_snake_case     		=	['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
								_snake_case     		=	tokenizer.tokenize(lowerCAmelCase_						)
								self.assertListEqual(lowerCAmelCase_					,     lowerCAmelCase_						)

								_snake_case     		=	tokens + [tokenizer.unk_token]
								_snake_case     		=	[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_						)					,     lowerCAmelCase_						)


							def 				lowerCamelCase       ( self						):





								"""simple docstring"""






								_snake_case     		=	self.get_tokenizer()
								_snake_case     		=	tokenizer('Hello'					,     'World'						)
								_snake_case     		=	[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
								self.assertListEqual(tokd['token_type_ids']					,     lowerCAmelCase_						)


							@slow
							def 				lowerCamelCase       ( self						):





								"""simple docstring"""






								_snake_case     		=	self.tokenizer_class.from_pretrained('microsoft/deberta-base'						)

								_snake_case     		=	tokenizer.encode('sequence builders'					,     add_special_tokens=lowerCAmelCase_						)
								_snake_case     		=	tokenizer.encode('multi-sequence build'					,     add_special_tokens=lowerCAmelCase_						)

								_snake_case     		=	tokenizer.encode(
								    'sequence builders'					,     add_special_tokens=lowerCAmelCase_					,     add_prefix_space=lowerCAmelCase_						)
								_snake_case     		=	tokenizer.encode(
								    'sequence builders'					,     'multi-sequence build'					,     add_special_tokens=lowerCAmelCase_					,     add_prefix_space=lowerCAmelCase_						)

								_snake_case     		=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_						)
								_snake_case     		=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					,     lowerCAmelCase_						)

								assert encoded_sentence == encoded_text_from_decode
								assert encoded_pair == encoded_pair_from_decode




							@slow
							def 				lowerCamelCase       ( self						):





								"""simple docstring"""






								_snake_case     		=	[self.tokenizer_class]
								if self.test_rust_tokenizer:
									tokenizer_classes.append(self.rust_tokenizer_class						)

								for tokenizer_class in tokenizer_classes:
									_snake_case     		=	tokenizer_class.from_pretrained('microsoft/deberta-base'						)

									_snake_case     		=	[
									    'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
									    'ALBERT incorporates two parameter reduction techniques',
									    'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
									    ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
									    ' vocabulary embedding.',
									]

									_snake_case     		=	tokenizer(lowerCAmelCase_					,     padding=lowerCAmelCase_						)
									_snake_case     		=	[tokenizer.decode(lowerCAmelCase_					,     skip_special_tokens=lowerCAmelCase_						) for seq in encoding['input_ids']]

									# fmt: off
									_snake_case     		=	{
									    'input_ids': [
									        [1, 21_18, 1_11_26, 5_65, 35, 83, 2_51_91, 1_63, 1_88_54, 13, 1_21_56, 12, 1_61_01, 2_53_76, 1_38_07, 9, 2_22_05, 2_78_93, 16_35, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [1, 21_18, 1_11_26, 5_65, 2_45_36, 80, 4_37_97, 48_78, 73_73, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [1, 1_33, 78, 65, 16, 10, 37_24, 15_38, 3_31_83, 1_13_03, 4_37_97, 19_38, 4, 8_70, 2_41_65, 2_91_05, 5, 7_39, 3_26_44, 3_31_83, 1_13_03, 3_61_73, 88, 80, 6_50, 78_21, 4_59_40, 6, 52, 25_59, 5, 18_36, 9, 5, 73_97, 1_31_71, 31, 5, 18_36, 9, 3_26_44, 3_31_83, 1_13_03, 4, 2]
									    ],
									    'token_type_ids': [
									        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
									    ],
									    'attention_mask': [
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
									    ]
									}
									# fmt: on

									_snake_case     		=	[
									    'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
									    'ALBERT incorporates two parameter reduction techniques',
									    'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
									    ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
									    ' vocabulary embedding.',
									]

									self.assertDictEqual(encoding.data					,     lowerCAmelCase_						)

									for expected, decoded in zip(lowerCAmelCase_					,     lowerCAmelCase_						):
										self.assertEqual(lowerCAmelCase_					,     lowerCAmelCase_						)



'''simple docstring'''







from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
					if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")):
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline
else:
					from .pipeline_unclip import UnCLIPPipeline
					from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline
					from .text_proj import UnCLIPTextProjModel

