





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_mbart import MBartTokenizer
else:
      snake_case 				=       None


snake_case 				=       logging.get_logger(__name__)


snake_case 				=       {"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

snake_case 				=       {
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-en-ro""": """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json""",
        """facebook/mbart-large-cc25""": """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json""",
    },
}

snake_case 				=       {
    """facebook/mbart-large-en-ro""": 1_0_2_4,
    """facebook/mbart-large-cc25""": 1_0_2_4,
}

# fmt: off
snake_case 				=       ["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]





class 				A_				(      UpperCAmelCase	):







       """simple docstring"""





       SCREAMING_SNAKE_CASE_      : int      =  VOCAB_FILES_NAMES
       SCREAMING_SNAKE_CASE_      : Dict      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       SCREAMING_SNAKE_CASE_      : Tuple      =  PRETRAINED_VOCAB_FILES_MAP
       SCREAMING_SNAKE_CASE_      : Dict      =  ['''input_ids''', '''attention_mask''']
       SCREAMING_SNAKE_CASE_      : Optional[int]      =  MBartTokenizer

       SCREAMING_SNAKE_CASE_      : List[int]      =  []
       SCREAMING_SNAKE_CASE_      : List[int]      =  []







       def __init__(    self	:	int						,__A	:	List[str]=None						,__A	:	str=None						,__A	:	str="<s>"						,__A	:	Optional[int]="</s>"						,__A	:	Dict="</s>"						,__A	:	List[str]="<s>"						,__A	:	int="<unk>"						,__A	:	Optional[int]="<pad>"						,__A	:	Dict="<mask>"						,__A	:	Optional[Any]=None						,__A	:	Union[str, Any]=None						,__A	:	Optional[int]=None						,**__A	:	Optional[Any]						,)						->					Optional[int]:
         # Mask token behave like a normal word, i.e. include the space before it
         _lowercase							= AddedToken(__A						,lstrip=__A						,rstrip=__A  ) if isinstance(__A						,__A  ) else mask_token

         super().__init__(
             vocab_file=__A						,tokenizer_file=__A						,bos_token=__A						,eos_token=__A						,sep_token=__A						,cls_token=__A						,unk_token=__A						,pad_token=__A						,mask_token=__A						,src_lang=__A						,tgt_lang=__A						,additional_special_tokens=__A						,**__A						,)

         _lowercase							= vocab_file
         _lowercase							= False if not self.vocab_file else True

         _lowercase							= FAIRSEQ_LANGUAGE_CODES.copy()

         if additional_special_tokens is not None:
           # Only add those special tokens if they are not already there.
           _additional_special_tokens.extend(
               [t for t in additional_special_tokens if t not in _additional_special_tokens]  )

         self.add_special_tokens({'additional_special_tokens': _additional_special_tokens}  )
         _lowercase							= {
             lang_code: self.convert_tokens_to_ids(__A  ) for lang_code in FAIRSEQ_LANGUAGE_CODES
         }

         _lowercase							= src_lang if src_lang is not None else 'en_XX'
         _lowercase							= self.convert_tokens_to_ids(self._src_lang  )
         _lowercase							= tgt_lang
         self.set_src_lang_special_tokens(self._src_lang  )







       @property
       def    __UpperCAmelCase (    self	:	Union[str, Any]  )						->					str:
         return self._src_lang







       @src_lang.setter
       def    __UpperCAmelCase (    self	:	Optional[int]						,__A	:	str  )						->					None:
         _lowercase							= new_src_lang
         self.set_src_lang_special_tokens(self._src_lang  )







       def    __UpperCAmelCase (    self	:	Union[str, Any]						,__A	:	List[int]						,__A	:	Optional[List[int]] = None  )						->					List[int]:
         if token_ids_a is None:
           return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







       def    __UpperCAmelCase (    self	:	Optional[int]						,__A	:	List[int]						,__A	:	Optional[List[int]] = None  )						->					List[int]:
         _lowercase							= [self.sep_token_id]
         _lowercase							= [self.cls_token_id]

         if token_ids_a is None:
           return len(cls + token_ids_a + sep  ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]







       def    __UpperCAmelCase (    self	:	Optional[int]						,__A	:	Tuple						,__A	:	str						,__A	:	Optional[str]						,__A	:	Optional[str]						,**__A	:	Any  )						->					int:
         if src_lang is None or tgt_lang is None:
           raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'  )
         _lowercase							= src_lang
         _lowercase							= self(__A						,add_special_tokens=__A						,return_tensors=__A						,**__A  )
         _lowercase							= self.convert_tokens_to_ids(__A  )
         _lowercase							= tgt_lang_id
         return inputs







       def    __UpperCAmelCase (    self	:	int						,__A	:	List[str]						,__A	:	str = "en_XX"						,__A	:	Optional[List[str]] = None						,__A	:	str = "ro_RO"						,**__A	:	str						,)						->					BatchEncoding:
         _lowercase							= src_lang
         _lowercase							= tgt_lang
         return super().prepare_seqaseq_batch(__A						,__A						,**__A  )







       def    __UpperCAmelCase (    self	:	Tuple  )						->					Optional[int]:
         return self.set_src_lang_special_tokens(self.src_lang  )







       def    __UpperCAmelCase (    self	:	Union[str, Any]  )						->					Dict:
         return self.set_tgt_lang_special_tokens(self.tgt_lang  )







       def    __UpperCAmelCase (    self	:	Optional[Any]						,__A	:	Optional[int]  )						->					None:
         _lowercase							= self.convert_tokens_to_ids(__A  )
         _lowercase							= []
         _lowercase							= [self.eos_token_id, self.cur_lang_code]

         _lowercase							= self.convert_ids_to_tokens(self.prefix_tokens  )
         _lowercase							= self.convert_ids_to_tokens(self.suffix_tokens  )

         _lowercase							= processors.TemplateProcessing(
             single=prefix_tokens_str + ['$A'] + suffix_tokens_str						,pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str						,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,self.prefix_tokens + self.suffix_tokens  )  )						,)







       def    __UpperCAmelCase (    self	:	List[str]						,__A	:	str  )						->					None:
         _lowercase							= self.convert_tokens_to_ids(__A  )
         _lowercase							= []
         _lowercase							= [self.eos_token_id, self.cur_lang_code]

         _lowercase							= self.convert_ids_to_tokens(self.prefix_tokens  )
         _lowercase							= self.convert_ids_to_tokens(self.suffix_tokens  )

         _lowercase							= processors.TemplateProcessing(
             single=prefix_tokens_str + ['$A'] + suffix_tokens_str						,pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str						,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,self.prefix_tokens + self.suffix_tokens  )  )						,)







       def    __UpperCAmelCase (    self	:	Optional[Any]						,__A	:	str						,__A	:	Optional[str] = None  )						->					Tuple[str]:
         if not self.can_save_slow_tokenizer:
           raise ValueError(
               'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
               'tokenizer.'  )

         if not os.path.isdir(__A  ):
           logger.error(F"""Vocabulary path ({save_directory}) should be a directory."""  )
           return
         _lowercase							= os.path.join(
             __A						,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )

         if os.path.abspath(self.vocab_file  ) != os.path.abspath(__A  ):
           copyfile(self.vocab_file						,__A  )

         return (out_vocab_file,)












import inspect
import unittest

from transformers import RegNetConfig, is_flax_available
from transformers.testing_utils import require_flax, slow
from transformers.utils import cached_property, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor


if is_flax_available():
      import jax
      import jax.numpy as jnp

      from transformers.models.regnet.modeling_flax_regnet import FlaxRegNetForImageClassification, FlaxRegNetModel

if is_vision_available():
      from PIL import Image

      from transformers import AutoImageProcessor





class 				A_				(      unittest.TestCase	):







       """simple docstring"""





       def __init__(    self	:	List[str]						,__A	:	str						,__A	:	Dict=3						,__A	:	str=32						,__A	:	int=3						,__A	:	Union[str, Any]=10						,__A	:	Optional[Any]=[10, 20, 30, 40]						,__A	:	List[str]=[1, 1, 2, 1]						,__A	:	Optional[int]=True						,__A	:	Optional[Any]=True						,__A	:	int="relu"						,__A	:	Union[str, Any]=3						,__A	:	List[Any]=None						,)						->					int:
         _lowercase							= parent
         _lowercase							= batch_size
         _lowercase							= image_size
         _lowercase							= num_channels
         _lowercase							= embeddings_size
         _lowercase							= hidden_sizes
         _lowercase							= depths
         _lowercase							= is_training
         _lowercase							= use_labels
         _lowercase							= hidden_act
         _lowercase							= num_labels
         _lowercase							= scope
         _lowercase							= len(__A  )







       def    __UpperCAmelCase (    self	:	Optional[Any]  )						->					Any:
         _lowercase							= floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]  )

         _lowercase							= self.get_config()

         return config, pixel_values







       def    __UpperCAmelCase (    self	:	Any  )						->					int:
         return RegNetConfig(
             num_channels=self.num_channels						,embeddings_size=self.embeddings_size						,hidden_sizes=self.hidden_sizes						,depths=self.depths						,hidden_act=self.hidden_act						,num_labels=self.num_labels						,image_size=self.image_size						,)







       def    __UpperCAmelCase (    self	:	Tuple						,__A	:	Any						,__A	:	Union[str, Any]  )						->					List[Any]:
         _lowercase							= FlaxRegNetModel(config=__A  )
         _lowercase							= model(__A  )

         # Output shape (b, c, h, w)
         self.parent.assertEqual(
             result.last_hidden_state.shape						,(self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)						,)







       def    __UpperCAmelCase (    self	:	List[Any]						,__A	:	Optional[int]						,__A	:	Dict  )						->					Any:
         _lowercase							= self.num_labels
         _lowercase							= FlaxRegNetForImageClassification(config=__A  )
         _lowercase							= model(__A  )
         self.parent.assertEqual(result.logits.shape						,(self.batch_size, self.num_labels)  )







       def    __UpperCAmelCase (    self	:	Tuple  )						->					Any:
         _lowercase							= self.prepare_config_and_inputs()
         _lowercase						,		_lowercase							= config_and_inputs
         _lowercase							= {'pixel_values': pixel_values}
         return config, inputs_dict





@require_flax
class 				A_				(      UpperCAmelCase					,	unittest.TestCase	):







       """simple docstring"""





       SCREAMING_SNAKE_CASE_      : Tuple      =  (FlaxRegNetModel, FlaxRegNetForImageClassification) if is_flax_available() else ()

       SCREAMING_SNAKE_CASE_      : Union[str, Any]      =  False
       SCREAMING_SNAKE_CASE_      : Any      =  False
       SCREAMING_SNAKE_CASE_      : Union[str, Any]      =  False







       def    __UpperCAmelCase (    self	:	Dict  )						->					None:
         _lowercase							= FlaxRegNetModelTester(self  )
         _lowercase							= ConfigTester(self						,config_class=__A						,has_text_modality=__A  )







       def    __UpperCAmelCase (    self	:	List[str]  )						->					Union[str, Any]:
         self.create_and_test_config_common_properties()
         self.config_tester.create_and_test_config_to_json_string()
         self.config_tester.create_and_test_config_to_json_file()
         self.config_tester.create_and_test_config_from_and_save_pretrained()
         self.config_tester.create_and_test_config_with_num_labels()
         self.config_tester.check_config_can_be_init_without_params()
         self.config_tester.check_config_arguments_init()







       def    __UpperCAmelCase (    self	:	Optional[int]  )						->					Any:
         return







       def    __UpperCAmelCase (    self	:	Dict  )						->					int:
         _lowercase							= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*__A  )







       def    __UpperCAmelCase (    self	:	Optional[Any]  )						->					Any:
         _lowercase							= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_image_classification(*__A  )







       @unittest.skip(reason='RegNet does not use inputs_embeds'  )
       def    __UpperCAmelCase (    self	:	List[str]  )						->					Optional[int]:
         pass







       @unittest.skip(reason='RegNet does not support input and output embeddings'  )
       def    __UpperCAmelCase (    self	:	Tuple  )						->					Any:
         pass







       def    __UpperCAmelCase (    self	:	Dict  )						->					List[str]:
         _lowercase						,		_lowercase							= self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
           _lowercase							= model_class(__A  )
           _lowercase							= inspect.signature(model.__call__  )
           # signature.parameters is an OrderedDict => so arg_names order is deterministic
           _lowercase							= [*signature.parameters.keys()]

           _lowercase							= ['pixel_values']
           self.assertListEqual(arg_names[:1]						,__A  )







       def    __UpperCAmelCase (    self	:	Union[str, Any]  )						->					List[str]:
         def check_hidden_states_output(__A	:	Optional[Any]						,__A	:	Optional[int]						,__A	:	str  ):
           _lowercase							= model_class(__A  )

           _lowercase							= model(**self._prepare_for_class(__A						,__A  )  )

           _lowercase							= outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

           _lowercase							= self.model_tester.num_stages
           self.assertEqual(len(__A  )						,expected_num_stages + 1  )

         _lowercase						,		_lowercase							= self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
           _lowercase							= True
           check_hidden_states_output(__A						,__A						,__A  )

           # check that output_hidden_states also work using config
           del inputs_dict["output_hidden_states"]
           _lowercase							= True

           check_hidden_states_output(__A						,__A						,__A  )







       def    __UpperCAmelCase (    self	:	Optional[Any]  )						->					int:
         _lowercase						,		_lowercase							= self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
           with self.subTest(model_class.__name__  ):
             _lowercase							= self._prepare_for_class(__A						,__A  )
             _lowercase							= model_class(__A  )

             @jax.jit
             def model_jitted(__A	:	Any						,**__A	:	Union[str, Any]  ):
               return model(pixel_values=__A						,**__A  )

             with self.subTest('JIT Enabled'  ):
               _lowercase							= model_jitted(**__A  ).to_tuple()

             with self.subTest('JIT Disabled'  ):
               with jax.disable_jit():
                 _lowercase							= model_jitted(**__A  ).to_tuple()

             self.assertEqual(len(__A  )						,len(__A  )  )
             for jitted_output, output in zip(__A						,__A  ):
               self.assertEqual(jitted_output.shape						,output.shape  )

def 							SCREAMING_SNAKE_CASE__       (  )      ->						str:
  _lowercase							= Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'     )
  return image







@require_flax
class 				A_				(      unittest.TestCase	):







       """simple docstring"""





       @cached_property
       def    __UpperCAmelCase (    self	:	str  )						->					Any:
         return AutoImageProcessor.from_pretrained('facebook/regnet-y-040'  ) if is_vision_available() else None







       @slow
       def    __UpperCAmelCase (    self	:	Optional[Any]  )						->					Any:
         _lowercase							= FlaxRegNetForImageClassification.from_pretrained('facebook/regnet-y-040'  )

         _lowercase							= self.default_image_processor
         _lowercase							= prepare_img()
         _lowercase							= image_processor(images=__A						,return_tensors='np'  )

         _lowercase							= model(**__A  )

         # verify the logits
         _lowercase							= (1, 1000)
         self.assertEqual(outputs.logits.shape						,__A  )

         _lowercase							= jnp.array([-0.4180, -1.5051, -3.4836]  )

         self.assertTrue(jnp.allclose(outputs.logits[0, :3]						,__A						,atol=1e-4  )  )