"""simple docstring"""




import json
import logging
import os
import socket

import git
import numpy as np
import torch


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - PID: %(process)d -  %(message)s",
    datefmt="%m/%d/%Y %H:%M:%S",
    level=logging.INFO,
)
__lowerCamelCase											=  logging.getLogger(__name__)






def   UpperCAmelCase					(	UpperCamelCase__				):






							"""simple docstring"""





							A__	     =							git.Repo(search_parent_directories=UpperCamelCase__				)
							A__	     =							{
							    'repo_id': str(UpperCamelCase__				),
							    'repo_sha': str(repo.head.object.hexsha				),
							    'repo_branch': str(repo.active_branch				),
							}

							with open(os.path.join(UpperCamelCase__       ,    'git_log.json'				)       ,    'w'				) as f:
														json.dump(UpperCamelCase__       ,    UpperCamelCase__       ,    indent=4				)






def   UpperCAmelCase					(	UpperCamelCase__				):






							"""simple docstring"""





							if params.n_gpu <= 0:
														A__	     =							0
														A__	     =							-1
														A__	     =							True
														A__	     =							False
														return

							assert torch.cuda.is_available()

							logger.info('Initializing GPUs'				)
							if params.n_gpu > 1:
														assert params.local_rank != -1

														A__	     =							int(os.environ['WORLD_SIZE']				)
														A__	     =							int(os.environ['N_GPU_NODE']				)
														A__	     =							int(os.environ['RANK']				)

														# number of nodes / node ID
														A__	     =							params.world_size // params.n_gpu_per_node
														A__	     =							params.global_rank // params.n_gpu_per_node
														A__	     =							True

														assert params.n_nodes == int(os.environ['N_NODES']				)
														assert params.node_id == int(os.environ['NODE_RANK']				)

							# local job (single GPU)
							else:
														assert params.local_rank == -1

														A__	     =							1
														A__	     =							0
														A__	     =							0
														A__	     =							0
														A__	     =							1
														A__	     =							1
														A__	     =							False

							# sanity checks
							assert params.n_nodes >= 1
							assert 0 <= params.node_id < params.n_nodes
							assert 0 <= params.local_rank <= params.global_rank < params.world_size
							assert params.world_size == params.n_nodes * params.n_gpu_per_node

							# define whether this is the master process / if we are in multi-node distributed mode
							A__	     =							params.node_id == 0 and params.local_rank == 0
							A__	     =							params.n_nodes > 1

							# summary
							A__	     =							F'''--- Global rank: {params.global_rank} - '''
							logger.info(PREFIX + 'Number of nodes: %i' % params.n_nodes				)
							logger.info(PREFIX + 'Node ID        : %i' % params.node_id				)
							logger.info(PREFIX + 'Local rank     : %i' % params.local_rank				)
							logger.info(PREFIX + 'World size     : %i' % params.world_size				)
							logger.info(PREFIX + 'GPUs per node  : %i' % params.n_gpu_per_node				)
							logger.info(PREFIX + 'Master         : %s' % str(params.is_master				)				)
							logger.info(PREFIX + 'Multi-node     : %s' % str(params.multi_node				)				)
							logger.info(PREFIX + 'Multi-GPU      : %s' % str(params.multi_gpu				)				)
							logger.info(PREFIX + 'Hostname       : %s' % socket.gethostname()				)

							# set GPU device
							torch.cuda.set_device(params.local_rank				)

							# initialize multi-GPU
							if params.multi_gpu:
														logger.info('Initializing PyTorch distributed'				)
														torch.distributed.init_process_group(
														    init_method='env://'       ,    backend='nccl'       ,    )






def   UpperCAmelCase					(	UpperCamelCase__				):






							"""simple docstring"""





							np.random.seed(args.seed				)
							torch.manual_seed(args.seed				)
							if args.n_gpu > 0:
														torch.cuda.manual_seed_all(args.seed				)






"""simple docstring"""




import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin
@require_jieba
class     UpperCamelCase__(			__A		,						unittest.TestCase ):
						lowerCAmelCase__  :						Optional[int]													=		CpmAntTokenizer
						lowerCAmelCase__  :						List[str]													=		False





						def 				snake_case__	(       self     )					->				Dict:
													super().setUp()

													A__	     =							[
													    '<d>',
													    '</d>',
													    '<s>',
													    '</s>',
													    '</_>',
													    '<unk>',
													    '<pad>',
													    '</n>',
													    '我',
													    '是',
													    'C',
													    'P',
													    'M',
													    'A',
													    'n',
													    't',
													]
													A__	     =							os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['vocab_file']     )
													with open(self.vocab_file		,'w'		,encoding='utf-8'     ) as vocab_writer:
																				vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )





						@tooslow
						def 				snake_case__	(       self     )					->				List[Any]:
													A__	     =							CpmAntTokenizer.from_pretrained('openbmb/cpm-ant-10b'     )
													A__	     =							'今天天气真好！'
													A__	     =							['今天', '天气', '真', '好', '！']
													A__	     =							tokenizer.tokenize(__UpperCAmelCase     )
													self.assertListEqual(__UpperCAmelCase		,__UpperCAmelCase     )
													A__	     =							'今天天气真好！'
													A__	     =							[tokenizer.bos_token] + tokens

													A__	     =							[6, 98_02, 1_49_62, 20_82, 8_31, 2_44]
													self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase     )		,__UpperCAmelCase     )

													A__	     =							tokenizer.decode(__UpperCAmelCase     )
													self.assertEqual(__UpperCAmelCase		,__UpperCAmelCase     )





