


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import AddedToken, PreTrainedTokenizerFast
from ...utils import logging


snake_case_  :	Tuple							  =							logging.get_logger(__name__)

snake_case_  :	int							  =							{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

snake_case_  :	Dict							  =							{
    '''vocab_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/tokenizer.json''',
    },
}

class   A__							(						UpperCamelCase__     ):




						def __init__(				self    :					Optional[Any] ,			_a    :					Tuple=None ,			_a    :					Union[str, Any]=None ,			_a    :					Dict=False ,			_a    :					int=True ,			_a    :					str=False ,			_a    :					int="<s>" ,			_a    :					int="</s>" ,			_a    :					Optional[int]="<unk>" ,			_a    :					Dict="<sep>" ,			_a    :					Optional[int]="<pad>" ,			_a    :					str="<cls>" ,			_a    :					List[Any]="<mask>" ,			_a    :					Optional[Any]=["<eop>", "<eod>"] ,			**_a    :					Any ,			)							->   Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =AddedToken(_a ,			lstrip=_a ,			rstrip=_a     ) if isinstance(_a ,			_a     ) else mask_token

										super().__init__(
										    vocab_file=_a ,			tokenizer_file=_a ,			do_lower_case=_a ,			remove_space=_a ,			keep_accents=_a ,			bos_token=_a ,			eos_token=_a ,			unk_token=_a ,			sep_token=_a ,			pad_token=_a ,			cls_token=_a ,			mask_token=_a ,			additional_special_tokens=_a ,			**_a ,			)

										_SCREAMING_SNAKE_CASE	     =3
										_SCREAMING_SNAKE_CASE	     =do_lower_case
										_SCREAMING_SNAKE_CASE	     =remove_space
										_SCREAMING_SNAKE_CASE	     =keep_accents
										_SCREAMING_SNAKE_CASE	     =vocab_file
										_SCREAMING_SNAKE_CASE	     =False if not self.vocab_file else True

										try:
														import jieba
										except ModuleNotFoundError as error:
														raise error.__class__(
														    '''You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '''
														    '''See https://pypi.org/project/jieba/ for installation.'''     )
										_SCREAMING_SNAKE_CASE	     =jieba
										_SCREAMING_SNAKE_CASE	     =str.maketrans(''' \n''' ,			'''\u2582\u2583'''     )




						def 							__UpperCamelCase (				self    :					Any ,			_a    :					List[int] ,			_a    :					Optional[List[int]] = None     )							->   List[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =[self.sep_token_id]
										_SCREAMING_SNAKE_CASE	     =[self.cls_token_id]
										if token_ids_a is None:
														return token_ids_a + sep + cls
										return token_ids_a + sep + token_ids_a + sep + cls




						def 							__UpperCamelCase (				self    :					Dict ,			_a    :					List[int] ,			_a    :					Optional[List[int]] = None     )							->   List[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =[self.sep_token_id]
										_SCREAMING_SNAKE_CASE	     =[2]

										if token_ids_a is None:
														return len(token_ids_a + sep     ) * [0] + cls_segment_id
										return len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1] + cls_segment_id




						def 							__UpperCamelCase (				self    :					str ,			_a    :					str ,			_a    :					Optional[str] = None     )							->   Tuple[str]:


										"""simple docstring"""





										if not self.can_save_slow_tokenizer:
														raise ValueError(
														    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
														    '''tokenizer.'''     )

										if not os.path.isdir(_a     ):
														logger.error(f"Vocabulary path ({save_directory}) should be a directory"     )
														return
										_SCREAMING_SNAKE_CASE	     =os.path.join(
										    _a ,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )

										if os.path.abspath(self.vocab_file     ) != os.path.abspath(_a     ):
														copyfile(self.vocab_file ,			_a     )

										return (out_vocab_file,)




						def 							__UpperCamelCase (				self    :					Dict ,			_a    :					str ,			*_a    :					Optional[int] ,			**_a    :					Any     )							->   int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =[
										    ''' '''.join([x.translate(self.translator     ) for x in self.jieba.cut(_a ,			cut_all=_a     )]     )
										    for text in batch_text_or_text_pairs
										]
										return super()._batch_encode_plus(_a ,			*_a ,			**_a     )




						def 							__UpperCamelCase (				self    :					List[Any] ,			*_a    :					Union[str, Any] ,			**_a    :					Any     )							->   Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =super()._decode(*_a ,			**_a     )
										_SCREAMING_SNAKE_CASE	     =text.replace(''' ''' ,			''''''     ).replace('''\u2582''' ,			''' '''     ).replace('''\u2583''' ,			'''\n'''     )
										return text



import copy
import inspect

from transformers.testing_utils import require_torch, torch_device
from transformers.utils.backbone_utils import BackboneType







@require_torch
class      __lowercase  :





							"""simple docstring"""







							_UpperCAmelCase     =		()
							_UpperCAmelCase     =		True






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= self.config_class

										# test default config
										SCREAMING_SNAKE_CASE_						:			Tuple       						= config_class()
										self.assertIsNotNone(lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			Dict       						= ['stem'] + [F'''stage{idx}''' for idx in range(1       ,		len(config.depths    ) + 1    )]
										self.assertEqual(config.stage_names       ,		lowerCAmelCase__    )
										self.assertTrue(set(config.out_features    ).issubset(set(config.stage_names    )    )    )

										# Test out_features and out_indices are correctly set
										# out_features and out_indices both None
										SCREAMING_SNAKE_CASE_						:			List[Any]       						= config_class(out_features=lowerCAmelCase__       ,		out_indices=lowerCAmelCase__    )
										self.assertEqual(config.out_features       ,		[config.stage_names[-1]]    )
										self.assertEqual(config.out_indices       ,		[len(config.stage_names    ) - 1]    )

										# out_features and out_indices both set
										SCREAMING_SNAKE_CASE_						:			str       						= config_class(out_features=['stem', 'stage1']       ,		out_indices=[0, 1]    )
										self.assertEqual(config.out_features       ,		['stem', 'stage1']    )
										self.assertEqual(config.out_indices       ,		[0, 1]    )

										# Only out_features set
										SCREAMING_SNAKE_CASE_						:			List[str]       						= config_class(out_features=['stage1', 'stage3']    )
										self.assertEqual(config.out_features       ,		['stage1', 'stage3']    )
										self.assertEqual(config.out_indices       ,		[1, 3]    )

										# Only out_indices set
										SCREAMING_SNAKE_CASE_						:			List[Any]       						= config_class(out_indices=[0, 2]    )
										self.assertEqual(config.out_features       ,		[config.stage_names[0], config.stage_names[2]]    )
										self.assertEqual(config.out_indices       ,		[0, 2]    )

										# Error raised when out_indices do not correspond to out_features
										with self.assertRaises(lowerCAmelCase__    ):
													SCREAMING_SNAKE_CASE_						:			str       						= config_class(out_features=['stage1', 'stage2']       ,		out_indices=[0, 2]    )






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						:			Tuple       						= self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
													SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= model_class(lowerCAmelCase__    )
													SCREAMING_SNAKE_CASE_						:			int       						= inspect.signature(model.forward    )
													# signature.parameters is an OrderedDict => so arg_names order is deterministic
													SCREAMING_SNAKE_CASE_						:			Any       						= [*signature.parameters.keys()]
													SCREAMING_SNAKE_CASE_						:			Optional[int]       						= ['pixel_values']
													self.assertListEqual(arg_names[:1]       ,		lowerCAmelCase__    )






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						:			str       						= self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
													SCREAMING_SNAKE_CASE_						:			Any       						= model_class(lowerCAmelCase__    )
													self.assertEqual(len(model.channels    )       ,		len(config.out_features    )    )
													SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= model.num_features
													SCREAMING_SNAKE_CASE_						:			Dict       						= [config.stage_names.index(lowerCAmelCase__    ) for feat in config.out_features]
													SCREAMING_SNAKE_CASE_						:			List[str]       						= [num_features[idx] for idx in out_indices]
													self.assertListEqual(model.channels       ,		lowerCAmelCase__    )

													SCREAMING_SNAKE_CASE_						:			Tuple       						= copy.deepcopy(lowerCAmelCase__    )
													SCREAMING_SNAKE_CASE_						:			Dict       						= None
													SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= model_class(lowerCAmelCase__    )
													self.assertEqual(len(model.channels    )       ,		1    )
													self.assertListEqual(model.channels       ,		[num_features[-1]]    )

													SCREAMING_SNAKE_CASE_						:			Dict       						= copy.deepcopy(lowerCAmelCase__    )
													SCREAMING_SNAKE_CASE_						:			Tuple       						= None
													SCREAMING_SNAKE_CASE_						:			int       						= model_class(lowerCAmelCase__    )
													self.assertEqual(len(model.channels    )       ,		1    )
													self.assertListEqual(model.channels       ,		[num_features[-1]]    )






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
													SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= model_class(lowerCAmelCase__    )
													model.to(lowerCAmelCase__    )
													model.eval()
													SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= model(**lowerCAmelCase__    )

													self.assertEqual(len(result.feature_maps    )       ,		len(config.out_features    )    )
													self.assertEqual(len(model.channels    )       ,		len(config.out_features    )    )
													self.assertEqual(len(result.feature_maps    )       ,		len(config.out_indices    )    )
													self.assertEqual(len(model.channels    )       ,		len(config.out_indices    )    )

													# Check output of last stage is taken if out_features=None, out_indices=None
													SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= copy.deepcopy(lowerCAmelCase__    )
													SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= None
													SCREAMING_SNAKE_CASE_						:			int       						= model_class(lowerCAmelCase__    )
													model.to(lowerCAmelCase__    )
													model.eval()
													SCREAMING_SNAKE_CASE_						:			List[Any]       						= model(**lowerCAmelCase__    )

													self.assertEqual(len(result.feature_maps    )       ,		1    )
													self.assertEqual(len(model.channels    )       ,		1    )

													SCREAMING_SNAKE_CASE_						:			Tuple       						= copy.deepcopy(lowerCAmelCase__    )
													SCREAMING_SNAKE_CASE_						:			int       						= None
													SCREAMING_SNAKE_CASE_						:			Dict       						= model_class(lowerCAmelCase__    )
													model.to(lowerCAmelCase__    )
													model.eval()
													SCREAMING_SNAKE_CASE_						:			List[str]       						= model(**lowerCAmelCase__    )

													self.assertEqual(len(result.feature_maps    )       ,		1    )
													self.assertEqual(len(model.channels    )       ,		1    )

													# Check backbone can be initialized with fresh weights
													SCREAMING_SNAKE_CASE_						:			Tuple       						= copy.deepcopy(lowerCAmelCase__    )
													SCREAMING_SNAKE_CASE_						:			Dict       						= False
													SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= model_class(lowerCAmelCase__    )
													model.to(lowerCAmelCase__    )
													model.eval()
													SCREAMING_SNAKE_CASE_						:			str       						= model(**lowerCAmelCase__    )






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= self.model_tester.prepare_config_and_inputs_for_common()

										for backbone_class in self.all_model_classes:
													SCREAMING_SNAKE_CASE_						:			List[str]       						= backbone_class(lowerCAmelCase__    )

													self.assertTrue(hasattr(lowerCAmelCase__       ,		'backbone_type'    )    )
													self.assertTrue(hasattr(lowerCAmelCase__       ,		'stage_names'    )    )
													self.assertTrue(hasattr(lowerCAmelCase__       ,		'num_features'    )    )
													self.assertTrue(hasattr(lowerCAmelCase__       ,		'out_indices'    )    )
													self.assertTrue(hasattr(lowerCAmelCase__       ,		'out_features'    )    )
													self.assertTrue(hasattr(lowerCAmelCase__       ,		'out_feature_channels'    )    )
													self.assertTrue(hasattr(lowerCAmelCase__       ,		'channels'    )    )

													self.assertIsInstance(backbone.backbone_type       ,		lowerCAmelCase__    )
													# Verify num_features has been initialized in the backbone init
													self.assertIsNotNone(backbone.num_features    )
													self.assertTrue(len(backbone.channels    ) == len(backbone.out_indices    )    )
													self.assertTrue(len(backbone.stage_names    ) == len(backbone.num_features    )    )
													self.assertTrue(len(backbone.channels    ) <= len(backbone.num_features    )    )
													self.assertTrue(len(backbone.out_feature_channels    ) == len(backbone.stage_names    )    )






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						:			List[str]       						= self.model_tester.prepare_config_and_inputs_for_common()
										SCREAMING_SNAKE_CASE_						:			Tuple       						= inputs_dict['pixel_values'].shape[0]

										for backbone_class in self.all_model_classes:
													SCREAMING_SNAKE_CASE_						:			Optional[int]       						= backbone_class(lowerCAmelCase__    )
													backbone.to(lowerCAmelCase__    )
													backbone.eval()

													SCREAMING_SNAKE_CASE_						:			Dict       						= backbone(**lowerCAmelCase__    )

													# Test default outputs and verify feature maps
													self.assertIsInstance(outputs.feature_maps       ,		lowerCAmelCase__    )
													self.assertTrue(len(outputs.feature_maps    ) == len(backbone.channels    )    )
													for feature_map, n_channels in zip(outputs.feature_maps       ,		backbone.channels    ):
																self.assertTrue(feature_map.shape[:2]       ,		(batch_size, n_channels)    )
													self.assertIsNone(outputs.hidden_states    )
													self.assertIsNone(outputs.attentions    )

													# Test output_hidden_states=True
													SCREAMING_SNAKE_CASE_						:			List[Any]       						= backbone(**lowerCAmelCase__       ,		output_hidden_states=lowerCAmelCase__    )
													self.assertIsNotNone(outputs.hidden_states    )
													self.assertTrue(len(outputs.hidden_states    )       ,		len(backbone.stage_names    )    )
													for hidden_state, n_channels in zip(outputs.hidden_states       ,		backbone.channels    ):
																self.assertTrue(hidden_state.shape[:2]       ,		(batch_size, n_channels)    )

													# Test output_attentions=True
													if self.has_attentions:
																SCREAMING_SNAKE_CASE_						:			Tuple       						= backbone(**lowerCAmelCase__       ,		output_attentions=lowerCAmelCase__    )
																self.assertIsNotNone(outputs.attentions    )





