


import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin
class  lowerCAmelCase_       (				snake_case__     ,    unittest.TestCase					):
							UpperCamelCase_		:Dict		      =    CTRLTokenizer
							UpperCamelCase_		:Any		      =    False
							UpperCamelCase_		:Dict		      =    False




							def 						__snake_case					(   self     :   Union[str, Any]	):
											super().setUp()

											# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
											lowerCAmelCase__      			=						["adapt", "re@@", "a@@", "apt", "c@@", "t", "<unk>"]
											lowerCAmelCase__      			=						dict(zip(__A ,							range(len(__A	)	)	)	)
											lowerCAmelCase__      			=						["#version: 0.2", "a p", "ap t</w>", "r e", "a d", "ad apt</w>", ""]
											lowerCAmelCase__      			=						{"unk_token": "<unk>"}

											lowerCAmelCase__      			=						os.path.join(self.tmpdirname ,							VOCAB_FILES_NAMES['''vocab_file''']	)
											lowerCAmelCase__      			=						os.path.join(self.tmpdirname ,							VOCAB_FILES_NAMES['''merges_file''']	)
											with open(self.vocab_file ,							'''w''' ,							encoding='''utf-8'''	) as fp:
															fp.write(json.dumps(__A	) + '''\n'''	)
											with open(self.merges_file ,							'''w''' ,							encoding='''utf-8'''	) as fp:
															fp.write('''\n'''.join(__A	)	)




							def 						__snake_case					(   self     :   Any ,							**SCREAMING_SNAKE_CASE_     :   Optional[int]	):
											kwargs.update(self.special_tokens_map	)
											return CTRLTokenizer.from_pretrained(self.tmpdirname ,							**__A	)




							def 						__snake_case					(   self     :   str ,							SCREAMING_SNAKE_CASE_     :   Any	):
											lowerCAmelCase__      			=						"adapt react readapt apt"
											lowerCAmelCase__      			=						"adapt react readapt apt"
											return input_text, output_text




							def 						__snake_case					(   self     :   List[Any]	):
											lowerCAmelCase__      			=						CTRLTokenizer(self.vocab_file ,							self.merges_file ,							**self.special_tokens_map	)
											lowerCAmelCase__      			=						"adapt react readapt apt"
											lowerCAmelCase__      			=						"adapt re@@ a@@ c@@ t re@@ adapt apt".split()
											lowerCAmelCase__      			=						tokenizer.tokenize(__A	)
											self.assertListEqual(__A ,							__A	)

											lowerCAmelCase__      			=						tokens + [tokenizer.unk_token]

											lowerCAmelCase__      			=						[0, 1, 2, 4, 5, 1, 0, 3, 6]
											self.assertListEqual(tokenizer.convert_tokens_to_ids(__A	) ,							__A	)













import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


_UpperCAmelCase			:		List[Any]   				=    logging.getLogger()
@unittest.skip('Temporarily disable the doc tests.'					)
@require_torch
@require_tf
@slow
class  lowerCAmelCase_       (				unittest.TestCase					):




							def 						__snake_case					(   self     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Path ,							SCREAMING_SNAKE_CASE_     :   Union[str, None] = None ,							SCREAMING_SNAKE_CASE_     :   Union[List[str], None] = None ,							SCREAMING_SNAKE_CASE_     :   Union[str, List[str], None] = None ,							SCREAMING_SNAKE_CASE_     :   bool = True ,							):
											lowerCAmelCase__      			=						[file for file in os.listdir(SCREAMING_SNAKE_CASE_	) if os.path.isfile(os.path.join(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)	)]

											if identifier is not None:
															lowerCAmelCase__      			=						[file for file in files if identifier in file]

											if n_identifier is not None:
															if isinstance(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	):
																			for n_ in n_identifier:
																							lowerCAmelCase__      			=						[file for file in files if n_ not in file]
															else:
																			lowerCAmelCase__      			=						[file for file in files if n_identifier not in file]

											lowerCAmelCase__      			=						ignore_files or []
											ignore_files.append('''__init__.py'''	)
											lowerCAmelCase__      			=						[file for file in files if file not in ignore_files]

											for file in files:
															# Open all files
															print('''Testing''' ,							SCREAMING_SNAKE_CASE_	)

															if only_modules:
																			lowerCAmelCase__      			=						file.split('''.'''	)[0]
																			try:
																							lowerCAmelCase__      			=						getattr(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)
																							lowerCAmelCase__      			=						doctest.DocTestSuite(SCREAMING_SNAKE_CASE_	)
																							lowerCAmelCase__      			=						unittest.TextTestRunner().run(SCREAMING_SNAKE_CASE_	)
																							self.assertIs(len(result.failures	) ,							0	)
																			except AttributeError:
																							logger.info(f'{module_identifier} is not a module.'	)
															else:
																			lowerCAmelCase__      			=						doctest.testfile(str('''..''' / directory / file	) ,							optionflags=doctest.ELLIPSIS	)
																			self.assertIs(result.failed ,							0	)




							def 						__snake_case					(   self     :   Optional[Any]	):
											lowerCAmelCase__      			=						Path('''src/transformers'''	)
											lowerCAmelCase__      			=						'''modeling'''
											lowerCAmelCase__      			=						[
											    '''modeling_ctrl.py''',
											    '''modeling_tf_ctrl.py''',
											]
											self.analyze_directory(SCREAMING_SNAKE_CASE_ ,							identifier=SCREAMING_SNAKE_CASE_ ,							ignore_files=SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Dict	):
											lowerCAmelCase__      			=						Path('''src/transformers'''	)
											lowerCAmelCase__      			=						'''tokenization'''
											self.analyze_directory(SCREAMING_SNAKE_CASE_ ,							identifier=SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Tuple	):
											lowerCAmelCase__      			=						Path('''src/transformers'''	)
											lowerCAmelCase__      			=						'''configuration'''
											self.analyze_directory(SCREAMING_SNAKE_CASE_ ,							identifier=SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Optional[Any]	):
											lowerCAmelCase__      			=						Path('''src/transformers'''	)
											lowerCAmelCase__      			=						['''configuration''', '''modeling''', '''tokenization''']
											self.analyze_directory(SCREAMING_SNAKE_CASE_ ,							n_identifier=SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Tuple	):
											lowerCAmelCase__      			=						Path('''docs/source'''	)
											lowerCAmelCase__      			=						['''favicon.ico''']
											self.analyze_directory(SCREAMING_SNAKE_CASE_ ,							ignore_files=SCREAMING_SNAKE_CASE_ ,							only_modules=SCREAMING_SNAKE_CASE_	)





