


import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 	__lowercase    (__SCREAMING_SNAKE_CASE	,       unittest.TestCase  ):





							"""simple docstring"""







							_UpperCAmelCase      =						BertTokenizer
							_UpperCAmelCase      =						BertTokenizerFast
							_UpperCAmelCase      =						True
							_UpperCAmelCase      =						True
							_UpperCAmelCase      =						filter_non_english






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										super().setUp()

										SCREAMING_SNAKE_CASE_		:     int					=					[
										    '[UNK]',
										    '[CLS]',
										    '[SEP]',
										    '[PAD]',
										    '[MASK]',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										SCREAMING_SNAKE_CASE_		:     int					=					os.path.join(self.tmpdirname			,						VOCAB_FILES_NAMES['vocab_file']       )
										with open(self.vocab_file			,						'w'			,						encoding='utf-8'       ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Any					=					'UNwant\u00E9d,running'
										SCREAMING_SNAKE_CASE_		:     Tuple					=					'unwanted, running'
										return input_text, output_text






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[Any]					=					self.tokenizer_class(self.vocab_file       )

										SCREAMING_SNAKE_CASE_		:     Dict					=					tokenizer.tokenize('UNwant\u00E9d,running'       )
										self.assertListEqual(lowerCAmelCase__			,						['un', '##want', '##ed', ',', 'runn', '##ing']       )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__       )			,						[9, 6, 7, 1_2, 1_0, 1_1]       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										if not self.test_rust_tokenizer:
													return

										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					self.get_tokenizer()
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					self.get_rust_tokenizer()

										SCREAMING_SNAKE_CASE_		:     Any					=					'UNwant\u00E9d,running'

										SCREAMING_SNAKE_CASE_		:     int					=					tokenizer.tokenize(lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     Dict					=					rust_tokenizer.tokenize(lowerCAmelCase__       )
										self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					rust_tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
										self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     Tuple					=					self.get_rust_tokenizer()
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					tokenizer.encode(lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     str					=					rust_tokenizer.encode(lowerCAmelCase__       )
										self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )

										# With lower casing
										SCREAMING_SNAKE_CASE_		:     Dict					=					self.get_tokenizer(do_lower_case=lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     Dict					=					self.get_rust_tokenizer(do_lower_case=lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     Any					=					'UNwant\u00E9d,running'

										SCREAMING_SNAKE_CASE_		:     Tuple					=					tokenizer.tokenize(lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     List[str]					=					rust_tokenizer.tokenize(lowerCAmelCase__       )
										self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     List[str]					=					tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					rust_tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
										self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     Tuple					=					self.get_rust_tokenizer()
										SCREAMING_SNAKE_CASE_		:     Any					=					tokenizer.encode(lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					rust_tokenizer.encode(lowerCAmelCase__       )
										self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[Any]					=					BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'       )			,						['ah', '\u535A', '\u63A8', 'zz']       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					BasicTokenizer(do_lower_case=lowerCAmelCase__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '       )			,						['hello', '!', 'how', 'are', 'you', '?']       )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )			,						['hello']       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[Any]					=					BasicTokenizer(do_lower_case=lowerCAmelCase__			,						strip_accents=lowerCAmelCase__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )			,						['hällo', '!', 'how', 'are', 'you', '?']       )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )			,						['h\u00E9llo']       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					BasicTokenizer(do_lower_case=lowerCAmelCase__			,						strip_accents=lowerCAmelCase__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )			,						['hallo', '!', 'how', 'are', 'you', '?']       )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )			,						['hello']       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					BasicTokenizer(do_lower_case=lowerCAmelCase__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )			,						['hallo', '!', 'how', 'are', 'you', '?']       )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )			,						['hello']       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					BasicTokenizer(do_lower_case=lowerCAmelCase__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '       )			,						['HeLLo', '!', 'how', 'Are', 'yoU', '?']       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     str					=					BasicTokenizer(do_lower_case=lowerCAmelCase__			,						strip_accents=lowerCAmelCase__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )			,						['HäLLo', '!', 'how', 'Are', 'yoU', '?']       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Tuple					=					BasicTokenizer(do_lower_case=lowerCAmelCase__			,						strip_accents=lowerCAmelCase__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )			,						['HaLLo', '!', 'how', 'Are', 'yoU', '?']       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Any					=					BasicTokenizer(do_lower_case=lowerCAmelCase__			,						never_split=['[UNK]']       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'       )			,						['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Tuple					=					BasicTokenizer()
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					'a\n\'ll !!to?\'d of, can\'t.'
										SCREAMING_SNAKE_CASE_		:     Tuple					=					['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
										self.assertListEqual(tokenizer.tokenize(lowerCAmelCase__       )			,						lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[str]					=					['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

										SCREAMING_SNAKE_CASE_		:     List[str]					=					{}
										for i, token in enumerate(lowerCAmelCase__       ):
													SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					i
										SCREAMING_SNAKE_CASE_		:     List[str]					=					WordpieceTokenizer(vocab=lowerCAmelCase__			,						unk_token='[UNK]'       )

										self.assertListEqual(tokenizer.tokenize(''       )			,						[]       )

										self.assertListEqual(tokenizer.tokenize('unwanted running'       )			,						['un', '##want', '##ed', 'runn', '##ing']       )

										self.assertListEqual(tokenizer.tokenize('unwantedX running'       )			,						['[UNK]', 'runn', '##ing']       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										self.assertTrue(_is_whitespace(' '       )       )
										self.assertTrue(_is_whitespace('\t'       )       )
										self.assertTrue(_is_whitespace('\r'       )       )
										self.assertTrue(_is_whitespace('\n'       )       )
										self.assertTrue(_is_whitespace('\u00A0'       )       )

										self.assertFalse(_is_whitespace('A'       )       )
										self.assertFalse(_is_whitespace('-'       )       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										self.assertTrue(_is_control('\u0005'       )       )

										self.assertFalse(_is_control('A'       )       )
										self.assertFalse(_is_control(' '       )       )
										self.assertFalse(_is_control('\t'       )       )
										self.assertFalse(_is_control('\r'       )       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										self.assertTrue(_is_punctuation('-'       )       )
										self.assertTrue(_is_punctuation('$'       )       )
										self.assertTrue(_is_punctuation('`'       )       )
										self.assertTrue(_is_punctuation('.'       )       )

										self.assertFalse(_is_punctuation('A'       )       )
										self.assertFalse(_is_punctuation(' '       )       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					self.get_tokenizer()
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(lowerCAmelCase__       ) for t in ['Test', '\xad', 'test']]			,						[['[UNK]'], [], ['[UNK]']]       )

										self.assertListEqual(
										    [rust_tokenizer.tokenize(lowerCAmelCase__       ) for t in ['Test', '\xad', 'test']]			,						[['[UNK]'], [], ['[UNK]']]       )






							@slow
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     str					=					self.tokenizer_class.from_pretrained('bert-base-uncased'       )

										SCREAMING_SNAKE_CASE_		:     str					=					tokenizer.encode('sequence builders'			,						add_special_tokens=lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     Dict					=					tokenizer.encode('multi-sequence build'			,						add_special_tokens=lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     int					=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     int					=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__			,						lowerCAmelCase__       )

										assert encoded_sentence == [1_0_1] + text + [1_0_2]
										assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																SCREAMING_SNAKE_CASE_		:     List[str]					=					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__			,						**lowerCAmelCase__       )

																SCREAMING_SNAKE_CASE_		:     int					=					F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					tokenizer_r.encode_plus(
																    lowerCAmelCase__			,						return_attention_mask=lowerCAmelCase__			,						return_token_type_ids=lowerCAmelCase__			,						return_offsets_mapping=lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__			,						)

																SCREAMING_SNAKE_CASE_		:     List[str]					=					tokenizer_r.do_lower_case if hasattr(lowerCAmelCase__			,						'do_lower_case'       ) else False
																SCREAMING_SNAKE_CASE_		:     int					=					(
																    [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), 'A'),
																        ((1, 2), ','),
																        ((3, 5), 'na'),
																        ((5, 6), '##ï'),
																        ((6, 8), '##ve'),
																        ((9, 1_5), tokenizer_r.mask_token),
																        ((1_6, 2_1), 'Allen'),
																        ((2_1, 2_3), '##NL'),
																        ((2_3, 2_4), '##P'),
																        ((2_5, 3_3), 'sentence'),
																        ((3_3, 3_4), '.'),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																    if not do_lower_case
																    else [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), 'a'),
																        ((1, 2), ','),
																        ((3, 8), 'naive'),
																        ((9, 1_5), tokenizer_r.mask_token),
																        ((1_6, 2_1), 'allen'),
																        ((2_1, 2_3), '##nl'),
																        ((2_3, 2_4), '##p'),
																        ((2_5, 3_3), 'sentence'),
																        ((3_3, 3_4), '.'),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																)

																self.assertEqual(
																    [e[1] for e in expected_results]			,						tokenizer_r.convert_ids_to_tokens(tokens['input_ids']       )       )
																self.assertEqual([e[0] for e in expected_results]			,						tokens['offset_mapping']       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					['的', '人', '有']
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					''.join(lowerCAmelCase__       )
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																SCREAMING_SNAKE_CASE_		:     str					=					True
																SCREAMING_SNAKE_CASE_		:     List[Any]					=					self.tokenizer_class.from_pretrained(lowerCAmelCase__			,						**lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     Optional[int]					=					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__			,						**lowerCAmelCase__       )

																SCREAMING_SNAKE_CASE_		:     Any					=					tokenizer_p.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     int					=					tokenizer_r.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )

																SCREAMING_SNAKE_CASE_		:     Optional[int]					=					tokenizer_r.convert_ids_to_tokens(lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     List[Any]					=					tokenizer_p.convert_ids_to_tokens(lowerCAmelCase__       )

																# it is expected that each Chinese character is not preceded by "##"
																self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )
																self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )

																SCREAMING_SNAKE_CASE_		:     Any					=					False
																SCREAMING_SNAKE_CASE_		:     Dict					=					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__			,						**lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     Any					=					self.tokenizer_class.from_pretrained(lowerCAmelCase__			,						**lowerCAmelCase__       )

																SCREAMING_SNAKE_CASE_		:     List[Any]					=					tokenizer_r.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     int					=					tokenizer_p.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )

																SCREAMING_SNAKE_CASE_		:     Any					=					tokenizer_r.convert_ids_to_tokens(lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     Any					=					tokenizer_p.convert_ids_to_tokens(lowerCAmelCase__       )

																# it is expected that only the first Chinese character is not preceded by "##".
																SCREAMING_SNAKE_CASE_		:     Optional[int]					=					[
																    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCAmelCase__       )
																]
																self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )
																self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )









import argparse

import torch
from torch import nn

from transformers import MBartConfig, MBartForConditionalGeneration






def 					UpperCamelCase   (      __magic_name__	:				Optional[Any]	)  ->    Tuple:
		"""simple docstring"""




		lowercase__	     =						[
		    """encoder.version""",
		    """decoder.version""",
		    """model.encoder.version""",
		    """model.decoder.version""",
		    """_float_tensor""",
		    """decoder.output_projection.weight""",
		]
		for k in ignore_keys:
				state_dict.pop(__magic_name__       ,       __magic_name__	)






def 					UpperCamelCase   (      __magic_name__	:				int	)  ->    Union[str, Any]:
		"""simple docstring"""




		lowercase__     ,      lowercase__	     =						emb.weight.shape
		lowercase__	     =						nn.Linear(__magic_name__       ,       __magic_name__       ,       bias=__magic_name__	)
		lowercase__	     =						emb.weight.data
		return lin_layer






def 					UpperCamelCase   (      __magic_name__	:				str       ,       __magic_name__	:				str="facebook/mbart-large-en-ro"       ,       __magic_name__	:				Any=False       ,       __magic_name__	:				int=False	)  ->    Optional[int]:
		"""simple docstring"""




		lowercase__	     =						torch.load(__magic_name__       ,       map_location="""cpu"""	)["""model"""]
		remove_ignore_keys_(__magic_name__	)
		lowercase__	     =						state_dict["""encoder.embed_tokens.weight"""].shape[0]

		lowercase__	     =						MBartConfig.from_pretrained(__magic_name__       ,       vocab_size=__magic_name__	)
		if mbart_aa and finetuned:
				lowercase__	     =						"""relu"""

		lowercase__	     =						state_dict["""decoder.embed_tokens.weight"""]
		lowercase__	     =						MBartForConditionalGeneration(__magic_name__	)
		model.model.load_state_dict(__magic_name__	)

		if finetuned:
				lowercase__	     =						make_linear_from_emb(model.model.shared	)

		return model


if __name__ == "__main__":
	A			:						Optional[Any]        =  argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    'fairseq_path', type=str, help='bart.large, bart.large.cnn or a path to a model.pt on local filesystem.'
	)
	parser.add_argument('pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
	parser.add_argument(
	    '--hf_config',
	    default='facebook/mbart-large-cc25',
	    type=str,
	    help='Which huggingface architecture to use: mbart-large',
	)
	parser.add_argument('--mbart_50', action='store_true', help='whether the model is mMART-50 checkpoint')
	parser.add_argument('--finetuned', action='store_true', help='whether the model is a fine-tuned checkpoint')
	A			:						Any        =  parser.parse_args()
	A			:						str        =  convert_fairseq_mbart_checkpoint_from_disk(
	    args.fairseq_path, hf_config_path=args.hf_config, finetuned=args.finetuned, mbart_aa=args.mbart_aa
	)
	model.save_pretrained(args.pytorch_dump_folder_path)

