


'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_mbart import MBartTokenizer
else:
 SCREAMING_SNAKE_CASE_      =			None


SCREAMING_SNAKE_CASE_      =			logging.get_logger(__name__)


SCREAMING_SNAKE_CASE_      =			{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE_      =			{
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-en-ro': 'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json',
        'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json',
    },
}

SCREAMING_SNAKE_CASE_      =			{
    'facebook/mbart-large-en-ro': 10_24,
    'facebook/mbart-large-cc25': 10_24,
}

# fmt: off
SCREAMING_SNAKE_CASE_      =			['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']


class  lowerCAmelCase_						(       snake_case__      ):





  """simple docstring"""



  a_       :Dict												=VOCAB_FILES_NAMES
  a_       :Dict												=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_       :Any												=PRETRAINED_VOCAB_FILES_MAP
  a_       :Tuple												=["""input_ids""", """attention_mask"""]
  a_       :int												=MBartTokenizer

  a_       :List[int]												=[]
  a_       :List[int]												=[]

  def __init__(					self     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							Any=None				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]=None				,							SCREAMING_SNAKE_CASE__     :							Optional[int]="<s>"				,							SCREAMING_SNAKE_CASE__     :							str="</s>"				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]="</s>"				,							SCREAMING_SNAKE_CASE__     :							List[str]="<s>"				,							SCREAMING_SNAKE_CASE__     :							List[str]="<unk>"				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]="<pad>"				,							SCREAMING_SNAKE_CASE__     :							Optional[int]="<mask>"				,							SCREAMING_SNAKE_CASE__     :							List[str]=None				,							SCREAMING_SNAKE_CASE__     :							List[Any]=None				,							SCREAMING_SNAKE_CASE__     :							int=None				,							**SCREAMING_SNAKE_CASE__     :							Dict				,							):





       '''simple docstring'''

       __a       =   AddedToken(SCREAMING_SNAKE_CASE__				,							lstrip=SCREAMING_SNAKE_CASE__				,							rstrip=SCREAMING_SNAKE_CASE__						) if isinstance(SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__						) else mask_token

       super().__init__(
           vocab_file=SCREAMING_SNAKE_CASE__				,							tokenizer_file=SCREAMING_SNAKE_CASE__				,							bos_token=SCREAMING_SNAKE_CASE__				,							eos_token=SCREAMING_SNAKE_CASE__				,							sep_token=SCREAMING_SNAKE_CASE__				,							cls_token=SCREAMING_SNAKE_CASE__				,							unk_token=SCREAMING_SNAKE_CASE__				,							pad_token=SCREAMING_SNAKE_CASE__				,							mask_token=SCREAMING_SNAKE_CASE__				,							src_lang=SCREAMING_SNAKE_CASE__				,							tgt_lang=SCREAMING_SNAKE_CASE__				,							additional_special_tokens=SCREAMING_SNAKE_CASE__				,							**SCREAMING_SNAKE_CASE__				,							)

       __a       =   vocab_file
       __a       =   False if not self.vocab_file else True

       __a       =   FAIRSEQ_LANGUAGE_CODES.copy()

       if additional_special_tokens is not None:
            # Only add those special tokens if they are not already there.
            _additional_special_tokens.extend(
                [t for t in additional_special_tokens if t not in _additional_special_tokens]						)

       self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}						)
       __a       =   {
           lang_code: self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__						) for lang_code in FAIRSEQ_LANGUAGE_CODES
       }

       __a       =   src_lang if src_lang is not None else """en_XX"""
       __a       =   self.convert_tokens_to_ids(self._src_lang						)
       __a       =   tgt_lang
       self.set_src_lang_special_tokens(self._src_lang						)

  @property
  def __a						(					self     :							Tuple						):





       '''simple docstring'''

       return self._src_lang

  @src_lang.setter
  def __a						(					self     :							str				,							SCREAMING_SNAKE_CASE__     :							str						):





       '''simple docstring'''

       __a       =   new_src_lang
       self.set_src_lang_special_tokens(self._src_lang						)

  def __a						(					self     :							Any				,							SCREAMING_SNAKE_CASE__     :							List[int]				,							SCREAMING_SNAKE_CASE__     :							Optional[List[int]] = None						):





       '''simple docstring'''

       if token_ids_a is None:
            return self.prefix_tokens + token_ids_a + self.suffix_tokens
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

  def __a						(					self     :							int				,							SCREAMING_SNAKE_CASE__     :							List[int]				,							SCREAMING_SNAKE_CASE__     :							Optional[List[int]] = None						):





       '''simple docstring'''

       __a       =   [self.sep_token_id]
       __a       =   [self.cls_token_id]

       if token_ids_a is None:
            return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]

  def __a						(					self     :							Optional[Any]				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							Optional[str]				,							SCREAMING_SNAKE_CASE__     :							Optional[str]				,							**SCREAMING_SNAKE_CASE__     :							str						):





       '''simple docstring'''

       if src_lang is None or tgt_lang is None:
            raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""						)
       __a       =   src_lang
       __a       =   self(SCREAMING_SNAKE_CASE__				,							add_special_tokens=SCREAMING_SNAKE_CASE__				,							return_tensors=SCREAMING_SNAKE_CASE__				,							**SCREAMING_SNAKE_CASE__						)
       __a       =   self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__						)
       __a       =   tgt_lang_id
       return inputs

  def __a						(					self     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							List[str]				,							SCREAMING_SNAKE_CASE__     :							str = "en_XX"				,							SCREAMING_SNAKE_CASE__     :							Optional[List[str]] = None				,							SCREAMING_SNAKE_CASE__     :							str = "ro_RO"				,							**SCREAMING_SNAKE_CASE__     :							List[Any]				,							):





       '''simple docstring'''

       __a       =   src_lang
       __a       =   tgt_lang
       return super().prepare_seqaseq_batch(SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__				,							**SCREAMING_SNAKE_CASE__						)

  def __a						(					self     :							Optional[int]						):





       '''simple docstring'''

       return self.set_src_lang_special_tokens(self.src_lang						)

  def __a						(					self     :							List[Any]						):





       '''simple docstring'''

       return self.set_tgt_lang_special_tokens(self.tgt_lang						)

  def __a						(					self     :							Any				,							SCREAMING_SNAKE_CASE__     :							str						):





       '''simple docstring'''

       __a       =   self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__						)
       __a       =   []
       __a       =   [self.eos_token_id, self.cur_lang_code]

       __a       =   self.convert_ids_to_tokens(self.prefix_tokens						)
       __a       =   self.convert_ids_to_tokens(self.suffix_tokens						)

       __a       =   processors.TemplateProcessing(
           single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str				,							pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str				,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,							self.prefix_tokens + self.suffix_tokens						)						)				,							)

  def __a						(					self     :							Any				,							SCREAMING_SNAKE_CASE__     :							str						):





       '''simple docstring'''

       __a       =   self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__						)
       __a       =   []
       __a       =   [self.eos_token_id, self.cur_lang_code]

       __a       =   self.convert_ids_to_tokens(self.prefix_tokens						)
       __a       =   self.convert_ids_to_tokens(self.suffix_tokens						)

       __a       =   processors.TemplateProcessing(
           single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str				,							pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str				,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,							self.prefix_tokens + self.suffix_tokens						)						)				,							)


  def __a						(					self     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							Optional[str] = None						):





       '''simple docstring'''

       if not self.can_save_slow_tokenizer:
            raise ValueError(
                """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                """tokenizer."""						)

       if not os.path.isdir(SCREAMING_SNAKE_CASE__						):
            logger.error(f'''Vocabulary path ({save_directory}) should be a directory.'''						)
            return
       __a       =   os.path.join(
           SCREAMING_SNAKE_CASE__				,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

       if os.path.abspath(self.vocab_file						) != os.path.abspath(SCREAMING_SNAKE_CASE__						):
            copyfile(self.vocab_file				,							SCREAMING_SNAKE_CASE__						)

       return (out_vocab_file,)










'''simple docstring'''







import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
SCREAMING_SNAKE_CASE_      =			{'LayoutLMv2Config', 'LayoutLMv3Config'}


@is_pipeline_test
class  lowerCAmelCase_						(       unittest.TestCase      ):





  """simple docstring"""



  a_       :List[Any]												=MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
  a_       :List[Any]												=TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

  if model_mapping is not None:
    a_       :List[Any]												={config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
  if tf_model_mapping is not None:
    a_       :Tuple												={
        config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
    }

  @require_torch
  def __a						(					self     :							int						):





       '''simple docstring'''

       __a       =   pipeline(
           task="""text-classification"""				,							model="""hf-internal-testing/tiny-random-distilbert"""				,							framework="""pt"""						)

       __a       =   text_classifier("""This is great !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """LABEL_0""", """score""": 0.5_0_4}]						)

       __a       =   text_classifier("""This is great !"""				,							top_k=2						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """LABEL_0""", """score""": 0.5_0_4}, {"""label""": """LABEL_1""", """score""": 0.4_9_6}]						)

       __a       =   text_classifier(["""This is great !""", """This is bad"""]				,							top_k=2						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[
               [{"""label""": """LABEL_0""", """score""": 0.5_0_4}, {"""label""": """LABEL_1""", """score""": 0.4_9_6}],
               [{"""label""": """LABEL_0""", """score""": 0.5_0_4}, {"""label""": """LABEL_1""", """score""": 0.4_9_6}],
           ]				,							)

       __a       =   text_classifier("""This is great !"""				,							top_k=1						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """LABEL_0""", """score""": 0.5_0_4}]						)

       # Legacy behavior
       __a       =   text_classifier("""This is great !"""				,							return_all_scores=SCREAMING_SNAKE_CASE__						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """LABEL_0""", """score""": 0.5_0_4}]						)

       __a       =   text_classifier("""This is great !"""				,							return_all_scores=SCREAMING_SNAKE_CASE__						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[[{"""label""": """LABEL_0""", """score""": 0.5_0_4}, {"""label""": """LABEL_1""", """score""": 0.4_9_6}]]						)

       __a       =   text_classifier(["""This is great !""", """Something else"""]				,							return_all_scores=SCREAMING_SNAKE_CASE__						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[
               [{"""label""": """LABEL_0""", """score""": 0.5_0_4}, {"""label""": """LABEL_1""", """score""": 0.4_9_6}],
               [{"""label""": """LABEL_0""", """score""": 0.5_0_4}, {"""label""": """LABEL_1""", """score""": 0.4_9_6}],
           ]				,							)

       __a       =   text_classifier(["""This is great !""", """Something else"""]				,							return_all_scores=SCREAMING_SNAKE_CASE__						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[
               {"""label""": """LABEL_0""", """score""": 0.5_0_4},
               {"""label""": """LABEL_0""", """score""": 0.5_0_4},
           ]				,							)

  @require_torch
  def __a						(					self     :							Optional[int]						):





       '''simple docstring'''

       import torch

       __a       =   pipeline(
           task="""text-classification"""				,							model="""hf-internal-testing/tiny-random-distilbert"""				,							framework="""pt"""				,							device=torch.device("""cpu"""						)				,							)

       __a       =   text_classifier("""This is great !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """LABEL_0""", """score""": 0.5_0_4}]						)

  @require_tf
  def __a						(					self     :							List[str]						):





       '''simple docstring'''

       __a       =   pipeline(
           task="""text-classification"""				,							model="""hf-internal-testing/tiny-random-distilbert"""				,							framework="""tf"""						)

       __a       =   text_classifier("""This is great !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """LABEL_0""", """score""": 0.5_0_4}]						)

  @slow
  @require_torch
  def __a						(					self     :							int						):





       '''simple docstring'''

       __a       =   pipeline("""text-classification"""						)

       __a       =   text_classifier("""This is great !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """POSITIVE""", """score""": 1.0}]						)
       __a       =   text_classifier("""This is bad !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """NEGATIVE""", """score""": 1.0}]						)
       __a       =   text_classifier("""Birds are a type of animal"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """POSITIVE""", """score""": 0.9_8_8}]						)

  @slow
  @require_tf
  def __a						(					self     :							Union[str, Any]						):





       '''simple docstring'''

       __a       =   pipeline("""text-classification"""				,							framework="""tf"""						)

       __a       =   text_classifier("""This is great !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """POSITIVE""", """score""": 1.0}]						)
       __a       =   text_classifier("""This is bad !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """NEGATIVE""", """score""": 1.0}]						)
       __a       =   text_classifier("""Birds are a type of animal"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """POSITIVE""", """score""": 0.9_8_8}]						)

  def __a						(					self     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							int				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							Tuple						):





       '''simple docstring'''

       __a       =   TextClassificationPipeline(model=SCREAMING_SNAKE_CASE__				,							tokenizer=SCREAMING_SNAKE_CASE__						)
       return text_classifier, ["HuggingFace is in", "This is another test"]


  def __a						(					self     :							Any				,							SCREAMING_SNAKE_CASE__     :							Any				,							SCREAMING_SNAKE_CASE__     :							Tuple						):





       '''simple docstring'''

       __a       =   text_classifier.model
       # Small inputs because BartTokenizer tiny has maximum position embeddings = 22
       __a       =   """HuggingFace is in"""
       __a       =   text_classifier(SCREAMING_SNAKE_CASE__						)

       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}]						)
       self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()						)

       __a       =   ["""HuggingFace is in """, """Paris is in France"""]
       __a       =   text_classifier(SCREAMING_SNAKE_CASE__						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}, {"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}]				,							)
       self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()						)
       self.assertTrue(outputs[1]["""label"""] in model.config.idalabel.values()						)

       # Forcing to get all results with `top_k=None`
       # This is NOT the legacy format
       __a       =   text_classifier(SCREAMING_SNAKE_CASE__				,							top_k=SCREAMING_SNAKE_CASE__						)
       __a       =   len(model.config.idalabel.values()						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[[{"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}] * N, [{"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}] * N]				,							)

       __a       =   {"""text""": """HuggingFace is in """, """text_pair""": """Paris is in France"""}
       __a       =   text_classifier(SCREAMING_SNAKE_CASE__						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							{"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}				,							)
       self.assertTrue(outputs["""label"""] in model.config.idalabel.values()						)

       # This might be used a text pair, but tokenizer + pipe interaction
       # makes it hard to understand that it's not using the pair properly
       # https://github.com/huggingface/transformers/issues/17305
       # We disabled this usage instead as it was outputting wrong outputs.
       __a       =   [["""HuggingFace is in """, """Paris is in France"""]]
       with self.assertRaises(SCREAMING_SNAKE_CASE__						):
            text_classifier(SCREAMING_SNAKE_CASE__						)

       # This used to be valid for doing text pairs
       # We're keeping it working because of backward compatibility
       __a       =   text_classifier([[["""HuggingFace is in """, """Paris is in France"""]]]						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}]				,							)
       self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()						)






