

'''simple docstring'''





import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin





class 		__SCREAMING_SNAKE_CASE       ( lowercase__	,       unittest.TestCase			):
				lowerCamelCase_							=						TransfoXLTokenizer
				lowerCamelCase_							=						False
				lowerCamelCase_							=						False







				def    lowerCamelCase_  (					self    :		Union[str, Any]      ):
										'''simple docstring'''
										super().setUp()

										lowercase   :       int								=[
										    '''<unk>''',
										    '''[CLS]''',
										    '''[SEP]''',
										    '''want''',
										    '''unwanted''',
										    '''wa''',
										    '''un''',
										    '''running''',
										    ''',''',
										    '''low''',
										    '''l''',
										]
										lowercase   :       List[Any]								=os.path.join(self.tmpdirname		,				VOCAB_FILES_NAMES['''vocab_file''']      )
										with open(self.vocab_file		,				'''w'''		,				encoding='''utf-8'''      ) as vocab_writer:
																vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )







				def    lowerCamelCase_  (					self    :		List[str]		,				**UpperCAmelCase__    :		Any      ):
										'''simple docstring'''
										lowercase   :       Tuple								=True
										return TransfoXLTokenizer.from_pretrained(self.tmpdirname		,				**UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		Union[str, Any]		,				UpperCAmelCase__    :		Union[str, Any]      ):
										'''simple docstring'''
										lowercase   :       Any								='''<unk> UNwanted , running'''
										lowercase   :       str								='''<unk> unwanted, running'''
										return input_text, output_text







				def    lowerCamelCase_  (					self    :		Union[str, Any]      ):
										'''simple docstring'''
										lowercase   :       int								=TransfoXLTokenizer(vocab_file=self.vocab_file		,				lower_case=UpperCAmelCase__      )

										lowercase   :       Tuple								=tokenizer.tokenize('''<unk> UNwanted , running'''      )
										self.assertListEqual(UpperCAmelCase__		,				['''<unk>''', '''unwanted''', ''',''', '''running''']      )

										self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__      )		,				[0, 4, 8, 7]      )







				def    lowerCamelCase_  (					self    :		int      ):
										'''simple docstring'''
										lowercase   :       int								=TransfoXLTokenizer(lower_case=UpperCAmelCase__      )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  '''      )		,				['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']      )







				def    lowerCamelCase_  (					self    :		Optional[Any]      ):
										'''simple docstring'''
										lowercase   :       List[str]								=TransfoXLTokenizer(lower_case=UpperCAmelCase__      )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  '''      )		,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )







				def    lowerCamelCase_  (					self    :		Tuple      ):
										'''simple docstring'''
										lowercase   :       List[str]								=TransfoXLTokenizer(lower_case=UpperCAmelCase__      )
										lowercase   :       Tuple								='''Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'''
										lowercase   :       Dict								=[
										    '''Hello''',
										    '''(''',
										    '''bracket''',
										    ''')''',
										    '''and''',
										    '''side''',
										    '''@-@''',
										    '''scrolled''',
										    '''[''',
										    '''and''',
										    ''']''',
										    '''Henry''',
										    '''\'s''',
										    '''$''',
										    '''5''',
										    '''@,@''',
										    '''000''',
										    '''with''',
										    '''3''',
										    '''@.@''',
										    '''34''',
										    '''m''',
										    '''.''',
										    '''What''',
										    '''\'s''',
										    '''up''',
										    '''!''',
										    '''?''',
										]

										self.assertListEqual(tokenizer.tokenize(UpperCAmelCase__      )		,				UpperCAmelCase__      )

										self.assertEqual(tokenizer.convert_tokens_to_string(UpperCAmelCase__      )		,				UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		List[Any]      ):
										'''simple docstring'''
										lowercase   :       Optional[int]								=self.get_tokenizer()
										lowercase   :       int								=len(UpperCAmelCase__      )

										tokenizer.add_tokens(['''new1''', '''new2''']      )
										tokenizer.move_added_token('''new1'''		,				1      )

										# Check that moved token is not copied (duplicate)
										self.assertEqual(len(UpperCAmelCase__      )		,				original_len + 2      )
										# Check that token is moved to specified id
										self.assertEqual(tokenizer.encode('''new1'''      )		,				[1]      )
										self.assertEqual(tokenizer.decode([1]      )		,				'''new1'''      )








'''simple docstring'''





# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import platform
import sys


UpperCamelCase_							=					"""3"""

print("""Python version:""", sys.version)

print("""OS platform:""", platform.platform())
print("""OS architecture:""", platform.machine())

try:
							import torch

							print("""Torch version:""", torch.__version__)
							print("""Cuda available:""", torch.cuda.is_available())
							print("""Cuda version:""", torch.version.cuda)
							print("""CuDNN version:""", torch.backends.cudnn.version())
							print("""Number of GPUs available:""", torch.cuda.device_count())
except ImportError:
							print("""Torch version:""", None)

try:
							import transformers

							print("""transformers version:""", transformers.__version__)
except ImportError:
							print("""transformers version:""", None)





