


'''simple docstring'''






import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 	a__       (							a__       ,       unittest.TestCase						):





							'''simple docstring'''






							lowercase__    :     Tuple					=   OpenAIGPTTokenizer
							lowercase__    :     Union[str, Any]					=   OpenAIGPTTokenizerFast
							lowercase__    :     Any					=   True
							lowercase__    :     List[Any]					=   False







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							str:
														super().setUp()

														# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
														lowerCAmelCase__      =			[
														    '''l''',
														    '''o''',
														    '''w''',
														    '''e''',
														    '''r''',
														    '''s''',
														    '''t''',
														    '''i''',
														    '''d''',
														    '''n''',
														    '''w</w>''',
														    '''r</w>''',
														    '''t</w>''',
														    '''lo''',
														    '''low''',
														    '''er</w>''',
														    '''low</w>''',
														    '''lowest</w>''',
														    '''newer</w>''',
														    '''wider</w>''',
														    '''<unk>''',
														]
														lowerCAmelCase__      =			dict(zip(lowerCamelCase_      ,					range(len(lowerCamelCase_     )     )     )     )
														lowerCAmelCase__      =			['''#version: 0.2''', '''l o''', '''lo w''', '''e r</w>''', '''''']

														lowerCAmelCase__      =			os.path.join(self.tmpdirname      ,					VOCAB_FILES_NAMES['''vocab_file''']     )
														lowerCAmelCase__      =			os.path.join(self.tmpdirname      ,					VOCAB_FILES_NAMES['''merges_file''']     )
														with open(self.vocab_file      ,					'''w'''     ) as fp:
																					fp.write(json.dumps(lowerCamelCase_     )     )
														with open(self.merges_file      ,					'''w'''     ) as fp:
																					fp.write('''\n'''.join(lowerCamelCase_     )     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							Optional[int]:
														return "lower newer", "lower newer"







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Tuple:
														lowerCAmelCase__      =			OpenAIGPTTokenizer(self.vocab_file      ,					self.merges_file     )

														lowerCAmelCase__      =			'''lower'''
														lowerCAmelCase__      =			['''low''', '''er</w>''']
														lowerCAmelCase__      =			tokenizer.tokenize(lowerCamelCase_     )
														self.assertListEqual(lowerCamelCase_      ,					lowerCamelCase_     )

														lowerCAmelCase__      =			tokens + ['''<unk>''']
														lowerCAmelCase__      =			[14, 15, 20]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_     )      ,					lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_=15     ) ->							Union[str, Any]:
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
																												lowerCAmelCase__      =			self.rust_tokenizer_class.from_pretrained(lowerCamelCase_      ,					**lowerCamelCase_     )

																												# Simple input
																												lowerCAmelCase__      =			'''This is a simple input'''
																												lowerCAmelCase__      =			['''This is a simple input 1''', '''This is a simple input 2''']
																												lowerCAmelCase__      =			('''This is a simple input''', '''This is a pair''')
																												lowerCAmelCase__      =			[
																												    ('''This is a simple input 1''', '''This is a simple input 2'''),
																												    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																												]

																												# Simple input tests
																												self.assertRaises(lowerCamelCase_      ,					tokenizer_r.encode      ,					lowerCamelCase_      ,					max_length=lowerCamelCase_      ,					padding='''max_length'''     )

																												# Simple input
																												self.assertRaises(lowerCamelCase_      ,					tokenizer_r.encode_plus      ,					lowerCamelCase_      ,					max_length=lowerCamelCase_      ,					padding='''max_length'''     )

																												# Simple input
																												self.assertRaises(
																												    lowerCamelCase_      ,					tokenizer_r.batch_encode_plus      ,					lowerCamelCase_      ,					max_length=lowerCamelCase_      ,					padding='''max_length'''      ,					)

																												# Pair input
																												self.assertRaises(lowerCamelCase_      ,					tokenizer_r.encode      ,					lowerCamelCase_      ,					max_length=lowerCamelCase_      ,					padding='''max_length'''     )

																												# Pair input
																												self.assertRaises(lowerCamelCase_      ,					tokenizer_r.encode_plus      ,					lowerCamelCase_      ,					max_length=lowerCamelCase_      ,					padding='''max_length'''     )

																												# Pair input
																												self.assertRaises(
																												    lowerCamelCase_      ,					tokenizer_r.batch_encode_plus      ,					lowerCamelCase_      ,					max_length=lowerCamelCase_      ,					padding='''max_length'''      ,					)







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Optional[Any]:
														pass




@require_ftfy
@require_spacy
@require_tokenizers
class 	a__       (							a__						):





							'''simple docstring'''






							pass



'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase					       =							logging.get_logger(__name__)

__UpperCAmelCase					       =							{'''ctrl''': '''https://huggingface.co/ctrl/resolve/main/config.json'''}




class 	a__       (							a__						):





							'''simple docstring'''






							lowercase__    :     Dict					=   "ctrl"
							lowercase__    :     str					=   ["past_key_values"]
							lowercase__    :     Optional[int]					=   {
							    "max_position_embeddings": "n_positions",
							    "hidden_size": "n_embd",
							    "num_attention_heads": "n_head",
							    "num_hidden_layers": "n_layer",
							}







							def __init__(	self      ,					lowerCamelCase_=24_65_34      ,					lowerCamelCase_=2_56      ,					lowerCamelCase_=12_80      ,					lowerCamelCase_=81_92      ,					lowerCamelCase_=48      ,					lowerCamelCase_=16      ,					lowerCamelCase_=0.1      ,					lowerCamelCase_=0.1      ,					lowerCamelCase_=1e-6      ,					lowerCamelCase_=0.02      ,					lowerCamelCase_=True      ,					**lowerCamelCase_      ,					) ->							Union[str, Any]:
														lowerCAmelCase__      =			vocab_size
														lowerCAmelCase__      =			n_positions
														lowerCAmelCase__      =			n_embd
														lowerCAmelCase__      =			n_layer
														lowerCAmelCase__      =			n_head
														lowerCAmelCase__      =			dff
														lowerCAmelCase__      =			resid_pdrop
														lowerCAmelCase__      =			embd_pdrop
														lowerCAmelCase__      =			layer_norm_epsilon
														lowerCAmelCase__      =			initializer_range

														lowerCAmelCase__      =			use_cache

														super().__init__(**lowerCamelCase_     )