


'''simple docstring'''






import json
import logging
import os
import socket

import git
import numpy as np
import torch


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - PID: %(process)d -  %(message)s',
    datefmt='%m/%d/%Y %H:%M:%S',
    level=logging.INFO,
)
__A	:	Tuple     =     logging.getLogger(__name__)







def      UpperCAmelCase      (		lowerCamelCase_  :str ):







		'''simple docstring'''
		snake_case_		:							str			       =						git.Repo(search_parent_directories=lowerCamelCase_ )
		snake_case_		:							List[Any]			       =						{
		    """repo_id""": str(lowerCamelCase_ ),
		    """repo_sha""": str(repo.head.object.hexsha ),
		    """repo_branch""": str(repo.active_branch ),
		}

		with open(os.path.join(lowerCamelCase_				,      """git_log.json""" )				,      """w""" ) as f:
				json.dump(lowerCamelCase_				,      lowerCamelCase_				,      indent=4 )







def      UpperCAmelCase      (		lowerCamelCase_  :List[str] ):







		'''simple docstring'''
		if params.n_gpu <= 0:
				snake_case_		:							Union[str, Any]			       =						0
				snake_case_		:							Optional[int]			       =						-1
				snake_case_		:							Any			       =						True
				snake_case_		:							Any			       =						False
				return

		assert torch.cuda.is_available()

		logger.info("""Initializing GPUs""" )
		if params.n_gpu > 1:
				assert params.local_rank != -1

				snake_case_		:							str			       =						int(os.environ["""WORLD_SIZE"""] )
				snake_case_		:							Optional[Any]			       =						int(os.environ["""N_GPU_NODE"""] )
				snake_case_		:							Union[str, Any]			       =						int(os.environ["""RANK"""] )

				# number of nodes / node ID
				snake_case_		:							List[Any]			       =						params.world_size // params.n_gpu_per_node
				snake_case_		:							Union[str, Any]			       =						params.global_rank // params.n_gpu_per_node
				snake_case_		:							int			       =						True

				assert params.n_nodes == int(os.environ["""N_NODES"""] )
				assert params.node_id == int(os.environ["""NODE_RANK"""] )

		# local job (single GPU)
		else:
				assert params.local_rank == -1

				snake_case_		:							Any			       =						1
				snake_case_		:							List[str]			       =						0
				snake_case_		:							Any			       =						0
				snake_case_		:							List[Any]			       =						0
				snake_case_		:							List[Any]			       =						1
				snake_case_		:							int			       =						1
				snake_case_		:							Optional[Any]			       =						False

		# sanity checks
		assert params.n_nodes >= 1
		assert 0 <= params.node_id < params.n_nodes
		assert 0 <= params.local_rank <= params.global_rank < params.world_size
		assert params.world_size == params.n_nodes * params.n_gpu_per_node

		# define whether this is the master process / if we are in multi-node distributed mode
		snake_case_		:							Optional[int]			       =						params.node_id == 0 and params.local_rank == 0
		snake_case_		:							List[Any]			       =						params.n_nodes > 1

		# summary
		snake_case_		:							Dict			       =						F'''--- Global rank: {params.global_rank} - '''
		logger.info(PREFIX + """Number of nodes: %i""" % params.n_nodes )
		logger.info(PREFIX + """Node ID        : %i""" % params.node_id )
		logger.info(PREFIX + """Local rank     : %i""" % params.local_rank )
		logger.info(PREFIX + """World size     : %i""" % params.world_size )
		logger.info(PREFIX + """GPUs per node  : %i""" % params.n_gpu_per_node )
		logger.info(PREFIX + """Master         : %s""" % str(params.is_master ) )
		logger.info(PREFIX + """Multi-node     : %s""" % str(params.multi_node ) )
		logger.info(PREFIX + """Multi-GPU      : %s""" % str(params.multi_gpu ) )
		logger.info(PREFIX + """Hostname       : %s""" % socket.gethostname() )

		# set GPU device
		torch.cuda.set_device(params.local_rank )

		# initialize multi-GPU
		if params.multi_gpu:
				logger.info("""Initializing PyTorch distributed""" )
				torch.distributed.init_process_group(
				    init_method="""env://"""				,      backend="""nccl"""				,      )







def      UpperCAmelCase      (		lowerCamelCase_  :List[str] ):







		'''simple docstring'''
		np.random.seed(args.seed )
		torch.manual_seed(args.seed )
		if args.n_gpu > 0:
				torch.cuda.manual_seed_all(args.seed )



'''simple docstring'''






import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin
class 						__UpperCamelCase	(						lowercase__						,  unittest.TestCase							):
							lowercase			:		Dict   		=  TransfoXLTokenizer
							lowercase			:		Optional[Any]   		=  False
							lowercase			:		Dict   		=  False





							def 		a__   (  self		:Union[str, Any]			):
									super().setUp()

									snake_case_		:							Optional[int]			       =						[
									    """<unk>""",
									    """[CLS]""",
									    """[SEP]""",
									    """want""",
									    """unwanted""",
									    """wa""",
									    """un""",
									    """running""",
									    """,""",
									    """low""",
									    """l""",
									]
									snake_case_		:							Optional[int]			       =						os.path.join(self.tmpdirname  ,VOCAB_FILES_NAMES["""vocab_file"""]			)
									with open(self.vocab_file  ,"""w"""  ,encoding="""utf-8"""			) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)





							def 		a__   (  self		:List[Any]  ,**_UpperCamelCase		:Optional[Any]			):
									snake_case_		:							Tuple			       =						True
									return TransfoXLTokenizer.from_pretrained(self.tmpdirname  ,**_UpperCamelCase			)





							def 		a__   (  self		:Tuple  ,_UpperCamelCase		:Union[str, Any]			):
									snake_case_		:							Any			       =						"""<unk> UNwanted , running"""
									snake_case_		:							Optional[int]			       =						"""<unk> unwanted, running"""
									return input_text, output_text





							def 		a__   (  self		:Dict			):
									snake_case_		:							Dict			       =						TransfoXLTokenizer(vocab_file=self.vocab_file  ,lower_case=_UpperCamelCase			)

									snake_case_		:							Dict			       =						tokenizer.tokenize("""<unk> UNwanted , running"""			)
									self.assertListEqual(_UpperCamelCase  ,["""<unk>""", """unwanted""", """,""", """running"""]			)

									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase			)  ,[0, 4, 8, 7]			)





							def 		a__   (  self		:Optional[Any]			):
									snake_case_		:							Dict			       =						TransfoXLTokenizer(lower_case=_UpperCamelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """			)  ,["""hello""", """!""", """how""", """are""", """you""", """?"""]			)





							def 		a__   (  self		:Any			):
									snake_case_		:							List[Any]			       =						TransfoXLTokenizer(lower_case=_UpperCamelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """			)  ,["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)





							def 		a__   (  self		:List[str]			):
									snake_case_		:							str			       =						TransfoXLTokenizer(lower_case=_UpperCamelCase			)
									snake_case_		:							List[str]			       =						"""Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
									snake_case_		:							Optional[int]			       =						[
									    """Hello""",
									    """(""",
									    """bracket""",
									    """)""",
									    """and""",
									    """side""",
									    """@-@""",
									    """scrolled""",
									    """[""",
									    """and""",
									    """]""",
									    """Henry""",
									    """'s""",
									    """$""",
									    """5""",
									    """@,@""",
									    """000""",
									    """with""",
									    """3""",
									    """@.@""",
									    """34""",
									    """m""",
									    """.""",
									    """What""",
									    """'s""",
									    """up""",
									    """!""",
									    """?""",
									]

									self.assertListEqual(tokenizer.tokenize(_UpperCamelCase			)  ,_UpperCamelCase			)

									self.assertEqual(tokenizer.convert_tokens_to_string(_UpperCamelCase			)  ,_UpperCamelCase			)





							def 		a__   (  self		:Dict			):
									snake_case_		:							Union[str, Any]			       =						self.get_tokenizer()
									snake_case_		:							Dict			       =						len(_UpperCamelCase			)

									tokenizer.add_tokens(["""new1""", """new2"""]			)
									tokenizer.move_added_token("""new1"""  ,1			)

									# Check that moved token is not copied (duplicate)
									self.assertEqual(len(_UpperCamelCase			)  ,original_len + 2			)
									# Check that token is moved to specified id
									self.assertEqual(tokenizer.encode("""new1"""			)  ,[1]			)
									self.assertEqual(tokenizer.decode([1]			)  ,"""new1"""			)