



'''simple docstring'''



import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_  :							List[Any]             =    get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
							from transformers.models.mbart.modeling_mbart import shift_tokens_right

lowerCAmelCase_  :							Tuple             =    25_0004
lowerCAmelCase_  :							Optional[int]             =    25_0020
@require_sentencepiece
@require_tokenizers
class 						lowerCamelCase_				(       a_     ,					unittest.TestCase      ):
						_lowerCAmelCase  : List[Any]						  =					MBartTokenizer
						_lowerCAmelCase  : Any						  =					MBartTokenizerFast
						_lowerCAmelCase  : List[Any]						  =					True
						_lowerCAmelCase  : Dict						  =					True







						def        __lowercase     (	self :   Union[str, Any]					):

													"""simple docstring"""
													super().setUp()

													# We have a SentencePiece fixture for testing
													SCREAMING_SNAKE_CASE       :   int         =      MBartTokenizer(lowercase_					,     keep_accents=lowercase_					)
													tokenizer.save_pretrained(self.tmpdirname					)







						def        __lowercase     (	self :   str					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   List[str]         =      MBartTokenizer(lowercase_					,     keep_accents=lowercase_					)

													SCREAMING_SNAKE_CASE       :   Dict         =      tokenizer.tokenize('''This is a test'''					)
													self.assertListEqual(lowercase_					,     ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']					)

													self.assertListEqual(
													    tokenizer.convert_tokens_to_ids(lowercase_					)					,     [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]					,     )

													SCREAMING_SNAKE_CASE       :   str         =      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''					)
													self.assertListEqual(
													    lowercase_					,     [
													        SPIECE_UNDERLINE + '''I''',
													        SPIECE_UNDERLINE + '''was''',
													        SPIECE_UNDERLINE + '''b''',
													        '''or''',
													        '''n''',
													        SPIECE_UNDERLINE + '''in''',
													        SPIECE_UNDERLINE + '''''',
													        '''9''',
													        '''2''',
													        '''0''',
													        '''0''',
													        '''0''',
													        ''',''',
													        SPIECE_UNDERLINE + '''and''',
													        SPIECE_UNDERLINE + '''this''',
													        SPIECE_UNDERLINE + '''is''',
													        SPIECE_UNDERLINE + '''f''',
													        '''al''',
													        '''s''',
													        '''é''',
													        '''.''',
													    ]					,     )
													SCREAMING_SNAKE_CASE       :   Tuple         =      tokenizer.convert_tokens_to_ids(lowercase_					)
													self.assertListEqual(
													    lowercase_					,     [
													        value + tokenizer.fairseq_offset
													        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
													        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
													    ]					,     )

													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      tokenizer.convert_ids_to_tokens(lowercase_					)
													self.assertListEqual(
													    lowercase_					,     [
													        SPIECE_UNDERLINE + '''I''',
													        SPIECE_UNDERLINE + '''was''',
													        SPIECE_UNDERLINE + '''b''',
													        '''or''',
													        '''n''',
													        SPIECE_UNDERLINE + '''in''',
													        SPIECE_UNDERLINE + '''''',
													        '''<unk>''',
													        '''2''',
													        '''0''',
													        '''0''',
													        '''0''',
													        ''',''',
													        SPIECE_UNDERLINE + '''and''',
													        SPIECE_UNDERLINE + '''this''',
													        SPIECE_UNDERLINE + '''is''',
													        SPIECE_UNDERLINE + '''f''',
													        '''al''',
													        '''s''',
													        '''<unk>''',
													        '''.''',
													    ]					,     )







						def        __lowercase     (	self :   str					):

													"""simple docstring"""
													if not self.test_slow_tokenizer:
																				# as we don't have a slow version, we can't compare the outputs between slow and fast versions
																				return

													SCREAMING_SNAKE_CASE       :   str         =      (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-mbart''', {})
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																				with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
																											SCREAMING_SNAKE_CASE       :   List[str]         =      self.rust_tokenizer_class.from_pretrained(lowercase_					,     **lowercase_					)
																											SCREAMING_SNAKE_CASE       :   Any         =      self.tokenizer_class.from_pretrained(lowercase_					,     **lowercase_					)

																											SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      tempfile.mkdtemp()

																											SCREAMING_SNAKE_CASE       :   List[str]         =      tokenizer_r.save_pretrained(lowercase_					)
																											SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      tokenizer_p.save_pretrained(lowercase_					)

																											# Checks it save with the same files + the tokenizer.json file for the fast one
																											self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files					)					)
																											SCREAMING_SNAKE_CASE       :   Optional[Any]         =      tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f					)
																											self.assertSequenceEqual(lowercase_					,     lowercase_					)

																											# Checks everything loads correctly in the same way
																											SCREAMING_SNAKE_CASE       :   Tuple         =      tokenizer_r.from_pretrained(lowercase_					)
																											SCREAMING_SNAKE_CASE       :   List[str]         =      tokenizer_p.from_pretrained(lowercase_					)

																											# Check special tokens are set accordingly on Rust and Python
																											for key in tokenizer_pp.special_tokens_map:
																																		self.assertTrue(hasattr(lowercase_					,     lowercase_					)					)
																																		# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																																		# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																											shutil.rmtree(lowercase_					)

																											# Save tokenizer rust, legacy_format=True
																											SCREAMING_SNAKE_CASE       :   Tuple         =      tempfile.mkdtemp()

																											SCREAMING_SNAKE_CASE       :   str         =      tokenizer_r.save_pretrained(lowercase_					,     legacy_format=lowercase_					)
																											SCREAMING_SNAKE_CASE       :   List[str]         =      tokenizer_p.save_pretrained(lowercase_					)

																											# Checks it save with the same files
																											self.assertSequenceEqual(lowercase_					,     lowercase_					)

																											# Checks everything loads correctly in the same way
																											SCREAMING_SNAKE_CASE       :   Optional[Any]         =      tokenizer_r.from_pretrained(lowercase_					)
																											SCREAMING_SNAKE_CASE       :   List[Any]         =      tokenizer_p.from_pretrained(lowercase_					)

																											# Check special tokens are set accordingly on Rust and Python
																											for key in tokenizer_pp.special_tokens_map:
																																		self.assertTrue(hasattr(lowercase_					,     lowercase_					)					)

																											shutil.rmtree(lowercase_					)

																											# Save tokenizer rust, legacy_format=False
																											SCREAMING_SNAKE_CASE       :   str         =      tempfile.mkdtemp()

																											SCREAMING_SNAKE_CASE       :   Tuple         =      tokenizer_r.save_pretrained(lowercase_					,     legacy_format=lowercase_					)
																											SCREAMING_SNAKE_CASE       :   Optional[Any]         =      tokenizer_p.save_pretrained(lowercase_					)

																											# Checks it saved the tokenizer.json file
																											self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files					)					)

																											# Checks everything loads correctly in the same way
																											SCREAMING_SNAKE_CASE       :   Any         =      tokenizer_r.from_pretrained(lowercase_					)
																											SCREAMING_SNAKE_CASE       :   int         =      tokenizer_p.from_pretrained(lowercase_					)

																											# Check special tokens are set accordingly on Rust and Python
																											for key in tokenizer_pp.special_tokens_map:
																																		self.assertTrue(hasattr(lowercase_					,     lowercase_					)					)

																											shutil.rmtree(lowercase_					)


@require_torch
@require_sentencepiece
@require_tokenizers
class 						lowerCamelCase_				(       unittest.TestCase      ):
						_lowerCAmelCase  : int						  =					'facebook/mbart-large-en-ro'
						_lowerCAmelCase  : Optional[int]						  =					[
						    ' UN Chief Says There Is No Military Solution in Syria',
						    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
						]
						_lowerCAmelCase  : str						  =					[
						    'Şeful ONU declară că nu există o soluţie militară în Siria',
						    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
						    ' pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor'
						    ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
						]
						_lowerCAmelCase  : int						  =					[8_2_7_4, 1_2_7_8_7_3, 2_5_9_1_6, 7, 8_6_2_2, 2_0_7_1, 4_3_8, 6_7_4_8_5, 5_3, 1_8_7_8_9_5, 2_3, 5_1_7_1_2, 2, EN_CODE]







						@classmethod
						def        __lowercase     (	cls :   Union[str, Any]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   MBartTokenizer         =      MBartTokenizer.from_pretrained(
													    cls.checkpoint_name					,     src_lang='''en_XX'''					,     tgt_lang='''ro_RO'''					)
													SCREAMING_SNAKE_CASE       :   Tuple         =      1
													return cls







						def        __lowercase     (	self :   Dict					):

													"""simple docstring"""
													self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ar_AR''']					,     25_00_01					)
													self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''en_EN''']					,     25_00_04					)
													self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ro_RO''']					,     25_00_20					)







						def        __lowercase     (	self :   Dict					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   List[str]         =      self.tokenizer.batch_encode_plus(self.src_text					).input_ids[0]
													self.assertListEqual(self.expected_src_tokens					,     lowercase_					)







						def        __lowercase     (	self :   Dict					):

													"""simple docstring"""
													self.assertIn(lowercase_					,     self.tokenizer.all_special_ids					)
													SCREAMING_SNAKE_CASE       :   Dict         =      [RO_CODE, 8_84, 90_19, 96, 9, 9_16, 8_67_92, 36, 1_87_43, 1_55_96, 5, 2]
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      self.tokenizer.decode(lowercase_					,     skip_special_tokens=lowercase_					)
													SCREAMING_SNAKE_CASE       :   int         =      self.tokenizer.decode(generated_ids[1:]					,     skip_special_tokens=lowercase_					)
													self.assertEqual(lowercase_					,     lowercase_					)
													self.assertNotIn(self.tokenizer.eos_token					,     lowercase_					)







						def        __lowercase     (	self :   int					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Dict         =      ['''this is gunna be a long sentence ''' * 20]
													assert isinstance(src_text[0]					,     lowercase_					)
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      10
													SCREAMING_SNAKE_CASE       :   str         =      self.tokenizer(lowercase_					,     max_length=lowercase_					,     truncation=lowercase_					).input_ids[0]
													self.assertEqual(ids[-2]					,     2					)
													self.assertEqual(ids[-1]					,     lowercase_					)
													self.assertEqual(len(lowercase_					)					,     lowercase_					)







						def        __lowercase     (	self :   Optional[int]					):

													"""simple docstring"""
													self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']					)					,     [25_00_26, 25_00_01]					)







						def        __lowercase     (	self :   Optional[int]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   int         =      tempfile.mkdtemp()
													SCREAMING_SNAKE_CASE       :   str         =      self.tokenizer.fairseq_tokens_to_ids
													self.tokenizer.save_pretrained(lowercase_					)
													SCREAMING_SNAKE_CASE       :   Any         =      MBartTokenizer.from_pretrained(lowercase_					)
													self.assertDictEqual(new_tok.fairseq_tokens_to_ids					,     lowercase_					)







						@require_torch
						def        __lowercase     (	self :   str					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   List[Any]         =      self.tokenizer(self.src_text					,     text_target=self.tgt_text					,     padding=lowercase_					,     return_tensors='''pt'''					)
													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      shift_tokens_right(batch['''labels''']					,     self.tokenizer.pad_token_id					)

													# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
													assert batch.input_ids[1][-2:].tolist() == [2, EN_CODE]
													assert batch.decoder_input_ids[1][0].tolist() == RO_CODE
													assert batch.decoder_input_ids[1][-1] == 2
													assert batch.labels[1][-2:].tolist() == [2, RO_CODE]







						@require_torch
						def        __lowercase     (	self :   str					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Tuple         =      self.tokenizer(
													    self.src_text					,     text_target=self.tgt_text					,     padding=lowercase_					,     truncation=lowercase_					,     max_length=len(self.expected_src_tokens					)					,     return_tensors='''pt'''					,     )

													SCREAMING_SNAKE_CASE       :   List[str]         =      shift_tokens_right(batch['''labels''']					,     self.tokenizer.pad_token_id					)

													self.assertIsInstance(lowercase_					,     lowercase_					)

													self.assertEqual((2, 14)					,     batch.input_ids.shape					)
													self.assertEqual((2, 14)					,     batch.attention_mask.shape					)
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      batch.input_ids.tolist()[0]
													self.assertListEqual(self.expected_src_tokens					,     lowercase_					)
													self.assertEqual(2					,     batch.decoder_input_ids[0, -1]					)  # EOS
													# Test that special tokens are reset
													self.assertEqual(self.tokenizer.prefix_tokens					,     []					)
													self.assertEqual(self.tokenizer.suffix_tokens					,     [self.tokenizer.eos_token_id, EN_CODE]					)







						def        __lowercase     (	self :   List[str]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Tuple         =      self.tokenizer(self.src_text					,     padding=lowercase_					,     truncation=lowercase_					,     max_length=3					,     return_tensors='''pt'''					)
													SCREAMING_SNAKE_CASE       :   Any         =      self.tokenizer(
													    text_target=self.tgt_text					,     padding=lowercase_					,     truncation=lowercase_					,     max_length=10					,     return_tensors='''pt'''					)
													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      targets['''input_ids''']
													SCREAMING_SNAKE_CASE       :   List[str]         =      shift_tokens_right(lowercase_					,     self.tokenizer.pad_token_id					)

													self.assertEqual(batch.input_ids.shape[1]					,     3					)
													self.assertEqual(batch.decoder_input_ids.shape[1]					,     10					)







						@require_torch
						def        __lowercase     (	self :   Any					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   List[str]         =      self.tokenizer._build_translation_inputs(
													    '''A test'''					,     return_tensors='''pt'''					,     src_lang='''en_XX'''					,     tgt_lang='''ar_AR'''					)

													self.assertEqual(
													    nested_simplify(lowercase_					)					,     {
													        # A, test, EOS, en_XX
													        '''input_ids''': [[62, 30_34, 2, 25_00_04]],
													        '''attention_mask''': [[1, 1, 1, 1]],
													        # ar_AR
													        '''forced_bos_token_id''': 25_00_01,
													    }					,     )







"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__UpperCamelCase    : List[Any]				 =						logging.get_logger(__name__)

__UpperCamelCase    : Dict				 =						{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCamelCase    : List[Any]				 =						{
    '''vocab_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-german-cased''': '''https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt''',
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-german-cased''': (
            '''https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
    },
}

__UpperCamelCase    : Union[str, Any]				 =						{
    '''distilbert-base-uncased''': 5_1_2,
    '''distilbert-base-uncased-distilled-squad''': 5_1_2,
    '''distilbert-base-cased''': 5_1_2,
    '''distilbert-base-cased-distilled-squad''': 5_1_2,
    '''distilbert-base-german-cased''': 5_1_2,
    '''distilbert-base-multilingual-cased''': 5_1_2,
}


__UpperCamelCase    : int				 =						{
    '''distilbert-base-uncased''': {'''do_lower_case''': True},
    '''distilbert-base-uncased-distilled-squad''': {'''do_lower_case''': True},
    '''distilbert-base-cased''': {'''do_lower_case''': False},
    '''distilbert-base-cased-distilled-squad''': {'''do_lower_case''': False},
    '''distilbert-base-german-cased''': {'''do_lower_case''': False},
    '''distilbert-base-multilingual-cased''': {'''do_lower_case''': False},
}







class 							SCREAMING_SNAKE_CASE   (						a_			):






					"""simple docstring"""



					lowercase__												=			VOCAB_FILES_NAMES
					lowercase__												=			PRETRAINED_VOCAB_FILES_MAP
					lowercase__												=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase__												=			PRETRAINED_INIT_CONFIGURATION
					lowercase__												=			["input_ids", "attention_mask"]
					lowercase__												=			DistilBertTokenizer







					def __init__(			self     :							int  ,lowercase_     :							List[str]=None  ,lowercase_     :							int=None  ,lowercase_     :							str=True  ,lowercase_     :							Union[str, Any]="[UNK]"  ,lowercase_     :							List[str]="[SEP]"  ,lowercase_     :							List[str]="[PAD]"  ,lowercase_     :							List[str]="[CLS]"  ,lowercase_     :							List[str]="[MASK]"  ,lowercase_     :							Tuple=True  ,lowercase_     :							Tuple=None  ,**lowercase_     :							str  ,):
						super().__init__(
						    lowercase_  ,tokenizer_file=lowercase_  ,do_lower_case=lowercase_  ,unk_token=lowercase_  ,sep_token=lowercase_  ,pad_token=lowercase_  ,cls_token=lowercase_  ,mask_token=lowercase_  ,tokenize_chinese_chars=lowercase_  ,strip_accents=lowercase_  ,**lowercase_  ,)

						lowerCAmelCase__      :		Tuple    							=      json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
						if (
						    normalizer_state.get('''lowercase'''  ,lowercase_						) != do_lower_case
						    or normalizer_state.get('''strip_accents'''  ,lowercase_						) != strip_accents
						    or normalizer_state.get('''handle_chinese_chars'''  ,lowercase_						) != tokenize_chinese_chars
						):
							lowerCAmelCase__      :		Union[str, Any]    							=      getattr(lowercase_  ,normalizer_state.pop('''type'''						)						)
							lowerCAmelCase__      :		Optional[int]    							=      do_lower_case
							lowerCAmelCase__      :		List[str]    							=      strip_accents
							lowerCAmelCase__      :		int    							=      tokenize_chinese_chars
							lowerCAmelCase__      :		Dict    							=      normalizer_class(**lowercase_						)

						lowerCAmelCase__      :		Tuple    							=      do_lower_case







					def 			__lowerCAmelCase					(			self     :							str  ,lowercase_     :							Tuple  ,lowercase_     :							Optional[Any]=None						):
						lowerCAmelCase__      :		Any    							=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

						return output







					def 			__lowerCAmelCase					(			self     :							int  ,lowercase_     :							List[int]  ,lowercase_     :							Optional[List[int]] = None						):
						lowerCAmelCase__      :		Dict    							=      [self.sep_token_id]
						lowerCAmelCase__      :		Dict    							=      [self.cls_token_id]
						if token_ids_a is None:
							return len(cls + token_ids_a + sep						) * [0]
						return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]







					def 			__lowerCAmelCase					(			self     :							Any  ,lowercase_     :							str  ,lowercase_     :							Optional[str] = None						):
						lowerCAmelCase__      :		Any    							=      self._tokenizer.model.save(lowercase_  ,name=lowercase_						)
						return tuple(lowercase_						)




