
import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert import BertTokenizer


lowercase        =  logging.get_logger(__name__)

lowercase        =  {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase        =  {
    '''vocab_file''': {
        '''facebook/dpr-ctx_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-ctx_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-ctx_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-ctx_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowercase        =  {
    '''vocab_file''': {
        '''facebook/dpr-question_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-question_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-question_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-question_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowercase        =  {
    '''vocab_file''': {
        '''facebook/dpr-reader-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-reader-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-reader-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-reader-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}

lowercase        =  {
    '''facebook/dpr-ctx_encoder-single-nq-base''': 5_1_2,
    '''facebook/dpr-ctx_encoder-multiset-base''': 5_1_2,
}
lowercase        =  {
    '''facebook/dpr-question_encoder-single-nq-base''': 5_1_2,
    '''facebook/dpr-question_encoder-multiset-base''': 5_1_2,
}
lowercase        =  {
    '''facebook/dpr-reader-single-nq-base''': 5_1_2,
    '''facebook/dpr-reader-multiset-base''': 5_1_2,
}


lowercase        =  {
    '''facebook/dpr-ctx_encoder-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-ctx_encoder-multiset-base''': {'''do_lower_case''': True},
}
lowercase        =  {
    '''facebook/dpr-question_encoder-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-question_encoder-multiset-base''': {'''do_lower_case''': True},
}
lowercase        =  {
    '''facebook/dpr-reader-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-reader-multiset-base''': {'''do_lower_case''': True},
}





class       __A(				UpperCAmelCase       ):
					SCREAMING_SNAKE_CASE			    =    VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE			    =    CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE			    =    CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE			    =    CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION





class       __A(				UpperCAmelCase       ):
					SCREAMING_SNAKE_CASE			    =    VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE			    =    QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE			    =    QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE			    =    QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION
lowercase        =  collections.namedtuple(
    '''DPRSpanPrediction''', ['''span_score''', '''relevance_score''', '''doc_id''', '''start_index''', '''end_index''', '''text''']
)

lowercase        =  collections.namedtuple('''DPRReaderOutput''', ['''start_logits''', '''end_logits''', '''relevance_logits'''])


lowercase        =  r'''
    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.
    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),
    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`
    with the format:

    ```
    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>
    ```

    Args:
        questions (`str` or `List[str]`):
            The questions to be encoded. You can specify one question for many passages. In this case, the question
            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in
            `titles` or `texts`.
        titles (`str` or `List[str]`):
            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.
        texts (`str` or `List[str]`):
            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.
        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
            Activates and controls padding. Accepts the following values:

            - `True` or `\'longest\'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
              if provided).
            - `\'max_length\'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided.
            - `False` or `\'do_not_pad\'` (default): No padding (i.e., can output a batch with sequences of different
              lengths).
        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
            Activates and controls truncation. Accepts the following values:

            - `True` or `\'longest_first\'`: Truncate to a maximum length specified with the argument `max_length` or to
              the maximum acceptable input length for the model if that argument is not provided. This will truncate
              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch
              of pairs) is provided.
            - `\'only_first\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the first
              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `\'only_second\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the
              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `False` or `\'do_not_truncate\'` (default): No truncation (i.e., can output batch with sequence lengths
              greater than the model maximum admissible input size).
        max_length (`int`, *optional*):
                Controls the maximum length to use by one of the truncation/padding parameters.

                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
                is required by one of the truncation/padding parameters. If the model has no specific maximum input
                length (like XLNet) truncation/padding to a maximum length will be deactivated.
        return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `\'tf\'`: Return TensorFlow `tf.constant` objects.
                - `\'pt\'`: Return PyTorch `torch.Tensor` objects.
                - `\'np\'`: Return Numpy `np.ndarray` objects.
        return_attention_mask (`bool`, *optional*):
            Whether or not to return the attention mask. If not set, will return the attention mask according to the
            specific tokenizer\'s default, defined by the `return_outputs` attribute.

            [What are attention masks?](../glossary#attention-mask)

    Returns:
        `Dict[str, List[List[int]]]`: A dictionary with the following keys:

        - `input_ids`: List of token ids to be fed to a model.
        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.
    '''





@add_start_docstrings(UpperCAmelCase       )
class       __A:

					def __call__(	self					:     Optional[Any]   ,					__UpperCamelCase					:     Union[str, Any]   ,					__UpperCamelCase					:     Optional[str] = None   ,					__UpperCamelCase					:     Optional[str] = None   ,					__UpperCamelCase					:     Union[bool, str] = False   ,					__UpperCamelCase					:     Union[bool, str] = False   ,					__UpperCamelCase					:     Optional[int] = None   ,					__UpperCamelCase					:     Optional[Union[str, TensorType]] = None   ,					__UpperCamelCase					:     Optional[bool] = None   ,					**__UpperCamelCase					:     Any   ,					):
												if titles is None and texts is None:
																			return super().__call__(
																			    __UpperCamelCase   ,					padding=__UpperCamelCase   ,					truncation=__UpperCamelCase   ,					max_length=__UpperCamelCase   ,					return_tensors=__UpperCamelCase   ,					return_attention_mask=__UpperCamelCase   ,					**__UpperCamelCase   ,					)
												elif titles is None or texts is None:
																			lowerCamelCase_							  =					titles if texts is None else texts
																			return super().__call__(
																			    __UpperCamelCase   ,					__UpperCamelCase   ,					padding=__UpperCamelCase   ,					truncation=__UpperCamelCase   ,					max_length=__UpperCamelCase   ,					return_tensors=__UpperCamelCase   ,					return_attention_mask=__UpperCamelCase   ,					**__UpperCamelCase   ,					)
												lowerCamelCase_							  =					titles if not isinstance(__UpperCamelCase   ,					__UpperCamelCase   ) else [titles]
												lowerCamelCase_							  =					texts if not isinstance(__UpperCamelCase   ,					__UpperCamelCase   ) else [texts]
												lowerCamelCase_							  =					len(__UpperCamelCase   )
												lowerCamelCase_							  =					questions if not isinstance(__UpperCamelCase   ,					__UpperCamelCase   ) else [questions] * n_passages
												if len(__UpperCamelCase   ) != len(__UpperCamelCase   ):
																			raise ValueError(
																			    F'''There should be as many titles than texts but got {len(__UpperCamelCase   )} titles and {len(__UpperCamelCase   )} texts.'''   )
												lowerCamelCase_							  =					super().__call__(__UpperCamelCase   ,					__UpperCamelCase   ,					padding=__UpperCamelCase   ,					truncation=__UpperCamelCase   )["""input_ids"""]
												lowerCamelCase_							  =					super().__call__(__UpperCamelCase   ,					add_special_tokens=__UpperCamelCase   ,					padding=__UpperCamelCase   ,					truncation=__UpperCamelCase   )["""input_ids"""]
												lowerCamelCase_							  =					{
												    """input_ids""": [
												        (encoded_question_and_title + encoded_text)[:max_length]
												        if max_length is not None and truncation
												        else encoded_question_and_title + encoded_text
												        for encoded_question_and_title, encoded_text in zip(__UpperCamelCase   ,					__UpperCamelCase   )
												    ]
												}
												if return_attention_mask is not False:
																			lowerCamelCase_							  =					[]
																			for input_ids in encoded_inputs["input_ids"]:
																										attention_mask.append([int(input_id != self.pad_token_id   ) for input_id in input_ids]   )
																			lowerCamelCase_							  =					attention_mask
												return self.pad(__UpperCamelCase   ,					padding=__UpperCamelCase   ,					max_length=__UpperCamelCase   ,					return_tensors=__UpperCamelCase   )

					def    lowercase__      (	self					:     Optional[int]   ,					__UpperCamelCase					:     BatchEncoding   ,					__UpperCamelCase					:     DPRReaderOutput   ,					__UpperCamelCase					:     int = 1_6   ,					__UpperCamelCase					:     int = 6_4   ,					__UpperCamelCase					:     int = 4   ,					):
												lowerCamelCase_							  =					reader_input["""input_ids"""]
												lowerCamelCase_  ,   lowerCamelCase_  ,   lowerCamelCase_							  =					reader_output[:3]
												lowerCamelCase_							  =					len(__UpperCamelCase   )
												lowerCamelCase_							  =					sorted(range(__UpperCamelCase   )   ,					reverse=__UpperCamelCase   ,					key=relevance_logits.__getitem__   )
												lowerCamelCase_							  =					[]
												for doc_id in sorted_docs:
																			lowerCamelCase_							  =					list(input_ids[doc_id]   )
																			# assuming question & title information is at the beginning of the sequence
																			lowerCamelCase_							  =					sequence_ids.index(self.sep_token_id   ,					2   ) + 1  # second sep id
																			if sequence_ids[-1] == self.pad_token_id:
																										lowerCamelCase_							  =					sequence_ids.index(self.pad_token_id   )
																			else:
																										lowerCamelCase_							  =					len(__UpperCamelCase   )

																			lowerCamelCase_							  =					self._get_best_spans(
																			    start_logits=start_logits[doc_id][passage_offset:sequence_len]   ,					end_logits=end_logits[doc_id][passage_offset:sequence_len]   ,					max_answer_length=__UpperCamelCase   ,					top_spans=__UpperCamelCase   ,					)
																			for start_index, end_index in best_spans:
																										start_index += passage_offset
																										end_index += passage_offset
																										nbest_spans_predictions.append(
																										    DPRSpanPrediction(
																										        span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]   ,					relevance_score=relevance_logits[doc_id]   ,					doc_id=__UpperCamelCase   ,					start_index=__UpperCamelCase   ,					end_index=__UpperCamelCase   ,					text=self.decode(sequence_ids[start_index : end_index + 1]   )   ,					)   )
																			if len(__UpperCamelCase   ) >= num_spans:
																										break
												return nbest_spans_predictions[:num_spans]



					def    lowercase__      (	self					:     Tuple   ,					__UpperCamelCase					:     List[int]   ,					__UpperCamelCase					:     List[int]   ,					__UpperCamelCase					:     int   ,					__UpperCamelCase					:     int   ,					):
												lowerCamelCase_							  =					[]
												for start_index, start_score in enumerate(__UpperCamelCase   ):
																			for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]   ):
																										scores.append(((start_index, start_index + answer_length), start_score + end_score)   )
												lowerCamelCase_							  =					sorted(__UpperCamelCase   ,					key=lambda __UpperCamelCase   : x[1]   ,					reverse=__UpperCamelCase   )
												lowerCamelCase_							  =					[]
												for (start_index, end_index), score in scores:
																			if start_index > end_index:
																										raise ValueError(F'''Wrong span indices: [{start_index}:{end_index}]'''   )
																			lowerCamelCase_							  =					end_index - start_index + 1
																			if length > max_answer_length:
																										raise ValueError(F'''Span is too long: {length} > {max_answer_length}'''   )
																			if any(
																			    start_index <= prev_start_index <= prev_end_index <= end_index
																			    or prev_start_index <= start_index <= end_index <= prev_end_index
																			    for (prev_start_index, prev_end_index) in chosen_span_intervals   ):
																										continue
																			chosen_span_intervals.append((start_index, end_index)   )

																			if len(__UpperCamelCase   ) == top_spans:
																										break
												return chosen_span_intervals





@add_end_docstrings(UpperCAmelCase       )
class       __A(				UpperCAmelCase     ,		UpperCAmelCase       ):
					SCREAMING_SNAKE_CASE			    =    VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE			    =    READER_PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE			    =    READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE			    =    READER_PRETRAINED_INIT_CONFIGURATION
					SCREAMING_SNAKE_CASE			    =    ['''input_ids''', '''attention_mask''']







import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowercase        =  get_tests_dir('''fixtures/test_sentencepiece.model''')

if is_torch_available():
		from transformers.models.mbart.modeling_mbart import shift_tokens_right

lowercase        =  2_5_0_0_0_4
lowercase        =  2_5_0_0_2_0





@require_sentencepiece
@require_tokenizers
class       __A(				UpperCAmelCase     ,		unittest.TestCase       ):
					SCREAMING_SNAKE_CASE			    =    MBartaaTokenizer
					SCREAMING_SNAKE_CASE			    =    MBartaaTokenizerFast
					SCREAMING_SNAKE_CASE			    =    True
					SCREAMING_SNAKE_CASE			    =    True

					def    lowercase__      (	self					:     Tuple   ):
												super().setUp()

												# We have a SentencePiece fixture for testing
												lowerCamelCase_							  =					MBartaaTokenizer(__UpperCamelCase   ,					src_lang="""en_XX"""   ,					tgt_lang="""ro_RO"""   ,					keep_accents=__UpperCamelCase   )
												tokenizer.save_pretrained(self.tmpdirname   )

					def    lowercase__      (	self					:     str   ):
												lowerCamelCase_							  =					"""<s>"""
												lowerCamelCase_							  =					0

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCamelCase   )   ,					__UpperCamelCase   )
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCamelCase   )   ,					__UpperCamelCase   )

					def    lowercase__      (	self					:     Optional[int]   ):
												lowerCamelCase_							  =					list(self.get_tokenizer().get_vocab().keys()   )

												self.assertEqual(vocab_keys[0]   ,					"""<s>"""   )
												self.assertEqual(vocab_keys[1]   ,					"""<pad>"""   )
												self.assertEqual(vocab_keys[-1]   ,					"""<mask>"""   )
												self.assertEqual(len(__UpperCamelCase   )   ,					1_0_5_4   )

					def    lowercase__      (	self					:     Optional[Any]   ):
												self.assertEqual(self.get_tokenizer().vocab_size   ,					1_0_5_4   )

					def    lowercase__      (	self					:     str   ):
												lowerCamelCase_							  =					MBartaaTokenizer(__UpperCamelCase   ,					src_lang="""en_XX"""   ,					tgt_lang="""ro_RO"""   ,					keep_accents=__UpperCamelCase   )

												lowerCamelCase_							  =					tokenizer.tokenize("""This is a test"""   )
												self.assertListEqual(__UpperCamelCase   ,					["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]   )

												self.assertListEqual(
												    tokenizer.convert_tokens_to_ids(__UpperCamelCase   )   ,					[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]   ,					)

												lowerCamelCase_							  =					tokenizer.tokenize("""I was born in 92000, and this is falsé."""   )
												self.assertListEqual(
												    __UpperCamelCase   ,					[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """9""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """é""", """."""]   ,					)
												lowerCamelCase_							  =					tokenizer.convert_tokens_to_ids(__UpperCamelCase   )
												self.assertListEqual(
												    __UpperCamelCase   ,					[
												        value + tokenizer.fairseq_offset
												        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
												    ]   ,					)

												lowerCamelCase_							  =					tokenizer.convert_ids_to_tokens(__UpperCamelCase   )
												self.assertListEqual(
												    __UpperCamelCase   ,					[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """<unk>""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """<unk>""", """."""]   ,					)

					@slow
					def    lowercase__      (	self					:     Optional[int]   ):
												# fmt: off
												lowerCamelCase_							  =					{"""input_ids""": [[2_5_0_0_0_4, 1_1_0_6_2, 8_2_7_7_2, 7, 1_5, 8_2_7_7_2, 5_3_8, 5_1_5_2_9, 2_3_7, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 2_1_5_1_7_5, 1_3_1_4, 1_3_6, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 5_6_3_5_9, 4_2, 1_2_2_0_0_9, 9, 1_6_4_6_6, 1_6, 8_7_3_4_4, 4_5_3_7, 9, 4_7_1_7, 7_8_3_8_1, 6, 1_5_9_9_5_8, 7, 1_5, 2_4_4_8_0, 6_1_8, 4, 5_2_7, 2_2_6_9_3, 5_4_2_8, 4, 2_7_7_7, 2_4_4_8_0, 9_8_7_4, 4, 4_3_5_2_3, 5_9_4, 4, 8_0_3, 1_8_3_9_2, 3_3_1_8_9, 1_8, 4, 4_3_5_2_3, 2_4_4_4_7, 1_2_3_9_9, 1_0_0, 2_4_9_5_5, 8_3_6_5_8, 9_6_2_6, 1_4_4_0_5_7, 1_5, 8_3_9, 2_2_3_3_5, 1_6, 1_3_6, 2_4_9_5_5, 8_3_6_5_8, 8_3_4_7_9, 1_5, 3_9_1_0_2, 7_2_4, 1_6, 6_7_8, 6_4_5, 2_7_8_9, 1_3_2_8, 4_5_8_9, 4_2, 1_2_2_0_0_9, 1_1_5_7_7_4, 2_3, 8_0_5, 1_3_2_8, 4_6_8_7_6, 7, 1_3_6, 5_3_8_9_4, 1_9_4_0, 4_2_2_2_7, 4_1_1_5_9, 1_7_7_2_1, 8_2_3, 4_2_5, 4, 2_7_5_1_2, 9_8_7_2_2, 2_0_6, 1_3_6, 5_5_3_1, 4_9_7_0, 9_1_9, 1_7_3_3_6, 5, 2], [2_5_0_0_0_4, 2_0_0_8_0, 6_1_8, 8_3, 8_2_7_7_5, 4_7, 4_7_9, 9, 1_5_1_7, 7_3, 5_3_8_9_4, 3_3_3, 8_0_5_8_1, 1_1_0_1_1_7, 1_8_8_1_1, 5_2_5_6, 1_2_9_5, 5_1, 1_5_2_5_2_6, 2_9_7, 7_9_8_6, 3_9_0, 1_2_4_4_1_6, 5_3_8, 3_5_4_3_1, 2_1_4, 9_8, 1_5_0_4_4, 2_5_7_3_7, 1_3_6, 7_1_0_8, 4_3_7_0_1, 2_3, 7_5_6, 1_3_5_3_5_5, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_5_0_0_0_4, 5_8_1, 6_3_7_7_3, 1_1_9_4_5_5, 6, 1_4_7_7_9_7, 8_8_2_0_3, 7, 6_4_5, 7_0, 2_1, 3_2_8_5, 1_0_2_6_9, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=__UpperCamelCase   ,					model_name="""facebook/mbart-large-50"""   ,					revision="""d3913889c59cd5c9e456b269c376325eabad57e2"""   ,					)



					def    lowercase__      (	self					:     Dict   ):
												if not self.test_slow_tokenizer:
																			# as we don't have a slow version, we can't compare the outputs between slow and fast versions
																			return

												lowerCamelCase_							  =					(self.rust_tokenizer_class, """hf-internal-testing/tiny-random-mbart50""", {})
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
																										lowerCamelCase_							  =					self.rust_tokenizer_class.from_pretrained(__UpperCamelCase   ,					**__UpperCamelCase   )
																										lowerCamelCase_							  =					self.tokenizer_class.from_pretrained(__UpperCamelCase   ,					**__UpperCamelCase   )

																										lowerCamelCase_							  =					tempfile.mkdtemp()

																										lowerCamelCase_							  =					tokenizer_r.save_pretrained(__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.save_pretrained(__UpperCamelCase   )

																										# Checks it save with the same files + the tokenizer.json file for the fast one
																										self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files   )   )
																										lowerCamelCase_							  =					tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f   )
																										self.assertSequenceEqual(__UpperCamelCase   ,					__UpperCamelCase   )

																										# Checks everything loads correctly in the same way
																										lowerCamelCase_							  =					tokenizer_r.from_pretrained(__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.from_pretrained(__UpperCamelCase   )

																										# Check special tokens are set accordingly on Rust and Python
																										for key in tokenizer_pp.special_tokens_map:
																																	self.assertTrue(hasattr(__UpperCamelCase   ,					__UpperCamelCase   )   )
																																	# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																																	# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																										shutil.rmtree(__UpperCamelCase   )

																										# Save tokenizer rust, legacy_format=True
																										lowerCamelCase_							  =					tempfile.mkdtemp()

																										lowerCamelCase_							  =					tokenizer_r.save_pretrained(__UpperCamelCase   ,					legacy_format=__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.save_pretrained(__UpperCamelCase   )

																										# Checks it save with the same files
																										self.assertSequenceEqual(__UpperCamelCase   ,					__UpperCamelCase   )

																										# Checks everything loads correctly in the same way
																										lowerCamelCase_							  =					tokenizer_r.from_pretrained(__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.from_pretrained(__UpperCamelCase   )

																										# Check special tokens are set accordingly on Rust and Python
																										for key in tokenizer_pp.special_tokens_map:
																																	self.assertTrue(hasattr(__UpperCamelCase   ,					__UpperCamelCase   )   )

																										shutil.rmtree(__UpperCamelCase   )

																										# Save tokenizer rust, legacy_format=False
																										lowerCamelCase_							  =					tempfile.mkdtemp()

																										lowerCamelCase_							  =					tokenizer_r.save_pretrained(__UpperCamelCase   ,					legacy_format=__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.save_pretrained(__UpperCamelCase   )

																										# Checks it saved the tokenizer.json file
																										self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files   )   )

																										# Checks everything loads correctly in the same way
																										lowerCamelCase_							  =					tokenizer_r.from_pretrained(__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.from_pretrained(__UpperCamelCase   )

																										# Check special tokens are set accordingly on Rust and Python
																										for key in tokenizer_pp.special_tokens_map:
																																	self.assertTrue(hasattr(__UpperCamelCase   ,					__UpperCamelCase   )   )

																										shutil.rmtree(__UpperCamelCase   )





@require_torch
@require_sentencepiece
@require_tokenizers
class       __A(				unittest.TestCase       ):
					SCREAMING_SNAKE_CASE			    =    '''facebook/mbart-large-50-one-to-many-mmt'''
					SCREAMING_SNAKE_CASE			    =    [
					    ''' UN Chief Says There Is No Military Solution in Syria''',
					    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
					]
					SCREAMING_SNAKE_CASE			    =    [
					    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
					    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
					    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
					    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
					]
					SCREAMING_SNAKE_CASE			    =    [EN_CODE, 8_2_7_4, 1_2_7_8_7_3, 2_5_9_1_6, 7, 8_6_2_2, 2_0_7_1, 4_3_8, 6_7_4_8_5, 5_3, 1_8_7_8_9_5, 2_3, 5_1_7_1_2, 2]

					@classmethod
					def    lowercase__      (	cls					:     Optional[Any]   ):
												lowerCamelCase_							  =					MBartaaTokenizer.from_pretrained(
												    cls.checkpoint_name   ,					src_lang="""en_XX"""   ,					tgt_lang="""ro_RO"""   )
												lowerCamelCase_							  =					1
												return cls

					def    lowercase__      (	self					:     Union[str, Any]   ):
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ar_AR"""]   ,					2_5_0_0_0_1   )
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""en_EN"""]   ,					2_5_0_0_0_4   )
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ro_RO"""]   ,					2_5_0_0_2_0   )
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""mr_IN"""]   ,					2_5_0_0_3_8   )

					def    lowercase__      (	self					:     Optional[Any]   ):
												lowerCamelCase_							  =					self.tokenizer.batch_encode_plus(self.src_text   ).input_ids[0]
												self.assertListEqual(self.expected_src_tokens   ,					__UpperCamelCase   )

					def    lowercase__      (	self					:     Tuple   ):
												self.assertIn(__UpperCamelCase   ,					self.tokenizer.all_special_ids   )
												lowerCamelCase_							  =					[RO_CODE, 8_8_4, 9_0_1_9, 9_6, 9, 9_1_6, 8_6_7_9_2, 3_6, 1_8_7_4_3, 1_5_5_9_6, 5, 2]
												lowerCamelCase_							  =					self.tokenizer.decode(__UpperCamelCase   ,					skip_special_tokens=__UpperCamelCase   )
												lowerCamelCase_							  =					self.tokenizer.decode(generated_ids[1:]   ,					skip_special_tokens=__UpperCamelCase   )
												self.assertEqual(__UpperCamelCase   ,					__UpperCamelCase   )
												self.assertNotIn(self.tokenizer.eos_token   ,					__UpperCamelCase   )

					def    lowercase__      (	self					:     Any   ):
												lowerCamelCase_							  =					["""this is gunna be a long sentence """ * 2_0]
												assert isinstance(src_text[0]   ,					__UpperCamelCase   )
												lowerCamelCase_							  =					1_0
												lowerCamelCase_							  =					self.tokenizer(__UpperCamelCase   ,					max_length=__UpperCamelCase   ,					truncation=__UpperCamelCase   ).input_ids[0]
												self.assertEqual(ids[0]   ,					__UpperCamelCase   )
												self.assertEqual(ids[-1]   ,					2   )
												self.assertEqual(len(__UpperCamelCase   )   ,					__UpperCamelCase   )

					def    lowercase__      (	self					:     Any   ):
												self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]   )   ,					[2_5_0_0_5_3, 2_5_0_0_0_1]   )

					def    lowercase__      (	self					:     Dict   ):
												lowerCamelCase_							  =					tempfile.mkdtemp()
												lowerCamelCase_							  =					self.tokenizer.fairseq_tokens_to_ids
												self.tokenizer.save_pretrained(__UpperCamelCase   )
												lowerCamelCase_							  =					MBartaaTokenizer.from_pretrained(__UpperCamelCase   )
												self.assertDictEqual(new_tok.fairseq_tokens_to_ids   ,					__UpperCamelCase   )

					@require_torch
					def    lowercase__      (	self					:     Optional[Any]   ):
												lowerCamelCase_							  =					self.tokenizer(self.src_text   ,					text_target=self.tgt_text   ,					padding=__UpperCamelCase   ,					return_tensors="""pt"""   )
												lowerCamelCase_							  =					shift_tokens_right(batch["""labels"""]   ,					self.tokenizer.pad_token_id   )

												# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
												assert batch.input_ids[1][0] == EN_CODE
												assert batch.input_ids[1][-1] == 2
												assert batch.labels[1][0] == RO_CODE
												assert batch.labels[1][-1] == 2
												assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]

					@require_torch
					def    lowercase__      (	self					:     Optional[int]   ):
												lowerCamelCase_							  =					self.tokenizer(
												    self.src_text   ,					text_target=self.tgt_text   ,					padding=__UpperCamelCase   ,					truncation=__UpperCamelCase   ,					max_length=len(self.expected_src_tokens   )   ,					return_tensors="""pt"""   ,					)
												lowerCamelCase_							  =					shift_tokens_right(batch["""labels"""]   ,					self.tokenizer.pad_token_id   )

												self.assertIsInstance(__UpperCamelCase   ,					__UpperCamelCase   )

												self.assertEqual((2, 1_4)   ,					batch.input_ids.shape   )
												self.assertEqual((2, 1_4)   ,					batch.attention_mask.shape   )
												lowerCamelCase_							  =					batch.input_ids.tolist()[0]
												self.assertListEqual(self.expected_src_tokens   ,					__UpperCamelCase   )
												self.assertEqual(2   ,					batch.decoder_input_ids[0, 0]   )  # decoder_start_token_id
												# Test that special tokens are reset
												self.assertEqual(self.tokenizer.prefix_tokens   ,					[EN_CODE]   )
												self.assertEqual(self.tokenizer.suffix_tokens   ,					[self.tokenizer.eos_token_id]   )

					def    lowercase__      (	self					:     Union[str, Any]   ):
												lowerCamelCase_							  =					self.tokenizer(self.src_text   ,					padding=__UpperCamelCase   ,					truncation=__UpperCamelCase   ,					max_length=3   ,					return_tensors="""pt"""   )
												lowerCamelCase_							  =					self.tokenizer(
												    text_target=self.tgt_text   ,					padding=__UpperCamelCase   ,					truncation=__UpperCamelCase   ,					max_length=1_0   ,					return_tensors="""pt"""   )
												lowerCamelCase_							  =					targets["""input_ids"""]
												lowerCamelCase_							  =					shift_tokens_right(__UpperCamelCase   ,					self.tokenizer.pad_token_id   )

												self.assertEqual(batch.input_ids.shape[1]   ,					3   )
												self.assertEqual(batch.decoder_input_ids.shape[1]   ,					1_0   )



					@require_torch
					def    lowercase__      (	self					:     Union[str, Any]   ):
												lowerCamelCase_							  =					self.tokenizer._build_translation_inputs(
												    """A test"""   ,					return_tensors="""pt"""   ,					src_lang="""en_XX"""   ,					tgt_lang="""ar_AR"""   )

												self.assertEqual(
												    nested_simplify(__UpperCamelCase   )   ,					{
												        # en_XX, A, test, EOS
												        """input_ids""": [[2_5_0_0_0_4, 6_2, 3_0_3_4, 2]],
												        """attention_mask""": [[1, 1, 1, 1]],
												        # ar_AR
												        """forced_bos_token_id""": 2_5_0_0_0_1,
												    }   ,					)





