





import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class       __A							(   _snake_case			,unittest.TestCase	):




      """simple docstring"""







      UpperCAmelCase__							=   None
      UpperCAmelCase__							=   BloomTokenizerFast
      UpperCAmelCase__							=   BloomTokenizerFast
      UpperCAmelCase__							=   True
      UpperCAmelCase__							=   False
      UpperCAmelCase__							=   """tokenizer_file"""
      UpperCAmelCase__							=   {"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}


      def 					__snake_case      (		self):



             """simple docstring"""





             super().setUp()
             _lowerCamelCase				:  int						=		BloomTokenizerFast.from_pretrained('''bigscience/tokenizer''')
             tokenizer.save_pretrained(self.tmpdirname)


      def 					__snake_case      (		self   ,				**a__):



             """simple docstring"""





             kwargs.update(self.special_tokens_map)
             return BloomTokenizerFast.from_pretrained(self.tmpdirname   ,				**__snake_case)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[int]						=		self.get_rust_tokenizer()

             _lowerCamelCase				:  int						=		["""The quick brown fox</s>""", """jumps over the lazy dog</s>"""]
             _lowerCamelCase				:  int						=		[[2175, 2_3714, 7_3173, 14_4252, 2], [77, 13_2619, 3478, 368, 10_9586, 3_5433, 2]]

             _lowerCamelCase				:  int						=		tokenizer.batch_encode_plus(__snake_case)["""input_ids"""]
             self.assertListEqual(__snake_case   ,				__snake_case)

             _lowerCamelCase				:  List[str]						=		tokenizer.batch_decode(__snake_case)
             self.assertListEqual(__snake_case   ,				__snake_case)


      def 					__snake_case      (		self   ,				a__=6):



             """simple docstring"""





             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                           _lowerCamelCase				:  Dict						=		self.rust_tokenizer_class.from_pretrained(__snake_case   ,				**__snake_case)
                           # tokenizer_r.pad_token = None # Hotfixing padding = None
                           # Simple input
                           _lowerCamelCase				:  Tuple						=		"""This is a simple input"""
                           _lowerCamelCase				:  Optional[int]						=		["""This is a simple input 1""", """This is a simple input 2"""]
                           _lowerCamelCase				:  str						=		("""This is a simple input""", """This is a pair""")
                           _lowerCamelCase				:  Optional[Any]						=		[
                               ("""This is a simple input 1""", """This is a simple input 2"""),
                               ("""This is a simple pair 1""", """This is a simple pair 2"""),
                           ]

                           # Simple input tests
                           try:
                                  tokenizer_r.encode(__snake_case   ,				max_length=__snake_case)
                                  tokenizer_r.encode_plus(__snake_case   ,				max_length=__snake_case)

                                  tokenizer_r.batch_encode_plus(__snake_case   ,				max_length=__snake_case)
                                  tokenizer_r.encode(__snake_case   ,				max_length=__snake_case)
                                  tokenizer_r.batch_encode_plus(__snake_case   ,				max_length=__snake_case)
                           except ValueError:
                                  self.fail('''Bloom Tokenizer should be able to deal with padding''')

                           _lowerCamelCase				:  str						=		None  # Hotfixing padding = None
                           self.assertRaises(__snake_case   ,				tokenizer_r.encode   ,				__snake_case   ,				max_length=__snake_case   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(__snake_case   ,				tokenizer_r.encode_plus   ,				__snake_case   ,				max_length=__snake_case   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(
                               __snake_case   ,				tokenizer_r.batch_encode_plus   ,				__snake_case   ,				max_length=__snake_case   ,				padding='''max_length'''   ,				)

                           # Pair input
                           self.assertRaises(__snake_case   ,				tokenizer_r.encode   ,				__snake_case   ,				max_length=__snake_case   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(__snake_case   ,				tokenizer_r.encode_plus   ,				__snake_case   ,				max_length=__snake_case   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(
                               __snake_case   ,				tokenizer_r.batch_encode_plus   ,				__snake_case   ,				max_length=__snake_case   ,				padding='''max_length'''   ,				)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  str						=		self.get_rust_tokenizer()
             _lowerCamelCase				:  List[str]						=		load_dataset('''xnli'''   ,				'''all_languages'''   ,				split='''test'''   ,				streaming=__snake_case)

             _lowerCamelCase				:  str						=		next(iter(__snake_case))["""premise"""]  # pick up one data
             _lowerCamelCase				:  Dict						=		list(sample_data.values())

             _lowerCamelCase				:  str						=		list(map(tokenizer.encode   ,				__snake_case))
             _lowerCamelCase				:  Any						=		[tokenizer.decode(__snake_case   ,				clean_up_tokenization_spaces=__snake_case) for x in output_tokens]
             self.assertListEqual(__snake_case   ,				__snake_case)



      def 					__snake_case      (		self):



             """simple docstring"""





             self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map)   ,				1)
             self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0])   ,				1)





'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
				from .tokenization_mbart import MBartTokenizer
else:
				UpperCAmelCase_						:  List[str]				  =						None


UpperCAmelCase_						:  Tuple				  =						logging.get_logger(__name__)


UpperCAmelCase_						:  List[Any]				  =						{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_						:  Dict				  =						{
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-en-ro': 'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json',
        'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json',
    },
}

UpperCAmelCase_						:  str				  =						{
    'facebook/mbart-large-en-ro': 1024,
    'facebook/mbart-large-cc25': 1024,
}

# fmt: off
UpperCAmelCase_						:  List[str]				  =						['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']





class        lowercase__    (		_snake_case   ):





							'''simple docstring'''


							A_				: Any       =    VOCAB_FILES_NAMES
							A_				: List[str]       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							A_				: Any       =    PRETRAINED_VOCAB_FILES_MAP
							A_				: Tuple       =    ["""input_ids""", """attention_mask"""]
							A_				: Dict       =    MBartTokenizer

							A_				: List[int]       =    []
							A_				: List[int]       =    []






							def __init__(       self       ,   __snake_case=None       ,   __snake_case=None       ,   __snake_case="<s>"       ,   __snake_case="</s>"       ,   __snake_case="</s>"       ,   __snake_case="<s>"       ,   __snake_case="<unk>"       ,   __snake_case="<pad>"       ,   __snake_case="<mask>"       ,   __snake_case=None       ,   __snake_case=None       ,   __snake_case=None       ,   **__snake_case       ,   ):
											# Mask token behave like a normal word, i.e. include the space before it
											_SCREAMING_SNAKE_CASE					:			List[Any]							= AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case    ) if isinstance(__snake_case       ,   __snake_case    ) else mask_token

											super().__init__(
											    vocab_file=__snake_case       ,   tokenizer_file=__snake_case       ,   bos_token=__snake_case       ,   eos_token=__snake_case       ,   sep_token=__snake_case       ,   cls_token=__snake_case       ,   unk_token=__snake_case       ,   pad_token=__snake_case       ,   mask_token=__snake_case       ,   src_lang=__snake_case       ,   tgt_lang=__snake_case       ,   additional_special_tokens=__snake_case       ,   **__snake_case       ,   )

											_SCREAMING_SNAKE_CASE					:			List[str]							= vocab_file
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= False if not self.vocab_file else True

											_SCREAMING_SNAKE_CASE					:			int							= FAIRSEQ_LANGUAGE_CODES.copy()

											if additional_special_tokens is not None:
															# Only add those special tokens if they are not already there.
															_additional_special_tokens.extend(
															    [t for t in additional_special_tokens if t not in _additional_special_tokens]    )

											self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}    )
											_SCREAMING_SNAKE_CASE					:			str							= {
											    lang_code: self.convert_tokens_to_ids(__snake_case    ) for lang_code in FAIRSEQ_LANGUAGE_CODES
											}

											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= src_lang if src_lang is not None else """en_XX"""
											_SCREAMING_SNAKE_CASE					:			str							= self.convert_tokens_to_ids(self._src_lang    )
											_SCREAMING_SNAKE_CASE					:			Any							= tgt_lang
											self.set_src_lang_special_tokens(self._src_lang    )






							@property
							def UpperCAmelCase_      (       self    ):
											return self._src_lang






							@src_lang.setter
							def UpperCAmelCase_      (       self       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			Any							= new_src_lang
											self.set_src_lang_special_tokens(self._src_lang    )






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = None    ):
											if token_ids_a is None:
															return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = None    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= [self.sep_token_id]
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= [self.cls_token_id]

											if token_ids_a is None:
															return len(cls + token_ids_a + sep    ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   **__snake_case    ):
											if src_lang is None or tgt_lang is None:
															raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""    )
											_SCREAMING_SNAKE_CASE					:			Dict							= src_lang
											_SCREAMING_SNAKE_CASE					:			List[Any]							= self(__snake_case       ,   add_special_tokens=__snake_case       ,   return_tensors=__snake_case       ,   **__snake_case    )
											_SCREAMING_SNAKE_CASE					:			Dict							= self.convert_tokens_to_ids(__snake_case    )
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= tgt_lang_id
											return inputs






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = "en_XX"       ,   __snake_case = None       ,   __snake_case = "ro_RO"       ,   **__snake_case       ,   ):
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= src_lang
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= tgt_lang
											return super().prepare_seqaseq_batch(__snake_case       ,   __snake_case       ,   **__snake_case    )






							def UpperCAmelCase_      (       self    ):
											return self.set_src_lang_special_tokens(self.src_lang    )






							def UpperCAmelCase_      (       self    ):
											return self.set_tgt_lang_special_tokens(self.tgt_lang    )






							def UpperCAmelCase_      (       self       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= self.convert_tokens_to_ids(__snake_case    )
											_SCREAMING_SNAKE_CASE					:			str							= []
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= [self.eos_token_id, self.cur_lang_code]

											_SCREAMING_SNAKE_CASE					:			int							= self.convert_ids_to_tokens(self.prefix_tokens    )
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= self.convert_ids_to_tokens(self.suffix_tokens    )

											_SCREAMING_SNAKE_CASE					:			Optional[int]							= processors.TemplateProcessing(
											    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str       ,   pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str       ,   special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,   self.prefix_tokens + self.suffix_tokens    )    )       ,   )






							def UpperCAmelCase_      (       self       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= self.convert_tokens_to_ids(__snake_case    )
											_SCREAMING_SNAKE_CASE					:			List[str]							= []
											_SCREAMING_SNAKE_CASE					:			Tuple							= [self.eos_token_id, self.cur_lang_code]

											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= self.convert_ids_to_tokens(self.prefix_tokens    )
											_SCREAMING_SNAKE_CASE					:			str							= self.convert_ids_to_tokens(self.suffix_tokens    )

											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= processors.TemplateProcessing(
											    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str       ,   pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str       ,   special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,   self.prefix_tokens + self.suffix_tokens    )    )       ,   )






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = None    ):
											if not self.can_save_slow_tokenizer:
															raise ValueError(
															    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
															    """tokenizer."""    )

											if not os.path.isdir(__snake_case    ):
															logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""    )
															return
											_SCREAMING_SNAKE_CASE					:			List[str]							= os.path.join(
											    __snake_case       ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

											if os.path.abspath(self.vocab_file    ) != os.path.abspath(__snake_case    ):
															copyfile(self.vocab_file       ,   __snake_case    )

											return (out_vocab_file,)




