
'''simple docstring'''



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
     from .tokenization_mbart import MBartTokenizer
else:
     UpperCamelCase__   :		int											=	None


UpperCamelCase__   :		int											=	logging.get_logger(__name__)


UpperCamelCase__   :		Dict											=	{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

UpperCamelCase__   :		Union[str, Any]											=	{
    "vocab_file": {
        "facebook/mbart-large-en-ro": (
            "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"
        ),
        "facebook/mbart-large-cc25": (
            "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-en-ro": "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json",
        "facebook/mbart-large-cc25": "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json",
    },
}

UpperCamelCase__   :		List[str]											=	{
    "facebook/mbart-large-en-ro": 1_024,
    "facebook/mbart-large-cc25": 1_024,
}

# fmt: off
UpperCamelCase__   :		int											=	["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN"]

class  _a					(lowercase_):





       """simple docstring"""




       SCREAMING_SNAKE_CASE   =	VOCAB_FILES_NAMES
       SCREAMING_SNAKE_CASE   =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       SCREAMING_SNAKE_CASE   =	PRETRAINED_VOCAB_FILES_MAP
       SCREAMING_SNAKE_CASE   =	['input_ids', 'attention_mask']
       SCREAMING_SNAKE_CASE   =	MBartTokenizer

       SCREAMING_SNAKE_CASE   =	[]
       SCREAMING_SNAKE_CASE   =	[]






       def __init__(				self							,				A__=None							,				A__=None							,				A__="<s>"							,				A__="</s>"							,				A__="</s>"							,				A__="<s>"							,				A__="<unk>"							,				A__="<pad>"							,				A__="<mask>"							,				A__=None							,				A__=None							,				A__=None							,				**A__							,				)     ->      List[Any]:
          # Mask token behave like a normal word, i.e. include the space before it
          _SCREAMING_SNAKE_CASE  			=					AddedToken(_lowercase							,				lstrip=_lowercase							,				rstrip=_lowercase   ) if isinstance(_lowercase							,				_lowercase   ) else mask_token

          super().__init__(
              vocab_file=_lowercase							,				tokenizer_file=_lowercase							,				bos_token=_lowercase							,				eos_token=_lowercase							,				sep_token=_lowercase							,				cls_token=_lowercase							,				unk_token=_lowercase							,				pad_token=_lowercase							,				mask_token=_lowercase							,				src_lang=_lowercase							,				tgt_lang=_lowercase							,				additional_special_tokens=_lowercase							,				**_lowercase							,				)

          _SCREAMING_SNAKE_CASE  			=					vocab_file
          _SCREAMING_SNAKE_CASE  			=					False if not self.vocab_file else True

          _SCREAMING_SNAKE_CASE  			=					FAIRSEQ_LANGUAGE_CODES.copy()

          if additional_special_tokens is not None:
             # Only add those special tokens if they are not already there.
             _additional_special_tokens.extend(
                 [t for t in additional_special_tokens if t not in _additional_special_tokens]   )

          self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}   )
          _SCREAMING_SNAKE_CASE  			=					{
              lang_code: self.convert_tokens_to_ids(_lowercase   ) for lang_code in FAIRSEQ_LANGUAGE_CODES
          }

          _SCREAMING_SNAKE_CASE  			=					src_lang if src_lang is not None else """en_XX"""
          _SCREAMING_SNAKE_CASE  			=					self.convert_tokens_to_ids(self._src_lang   )
          _SCREAMING_SNAKE_CASE  			=					tgt_lang
          self.set_src_lang_special_tokens(self._src_lang   )






       @property
       def 							UpperCamelCase					(				self   )     ->      str:
          return self._src_lang






       @src_lang.setter
       def 							UpperCamelCase					(				self							,				A__   )     ->      None:
          _SCREAMING_SNAKE_CASE  			=					new_src_lang
          self.set_src_lang_special_tokens(self._src_lang   )






       def 							UpperCamelCase					(				self							,				A__							,				A__ = None   )     ->      List[int]:
          if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + self.suffix_tokens
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






       def 							UpperCamelCase					(				self							,				A__							,				A__ = None   )     ->      List[int]:
          _SCREAMING_SNAKE_CASE  			=					[self.sep_token_id]
          _SCREAMING_SNAKE_CASE  			=					[self.cls_token_id]

          if token_ids_a is None:
             return len(cls + token_ids_a + sep   ) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]






       def 							UpperCamelCase					(				self							,				A__							,				A__							,				A__							,				A__							,				**A__   )     ->      int:
          if src_lang is None or tgt_lang is None:
             raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""   )
          _SCREAMING_SNAKE_CASE  			=					src_lang
          _SCREAMING_SNAKE_CASE  			=					self(_lowercase							,				add_special_tokens=_lowercase							,				return_tensors=_lowercase							,				**_lowercase   )
          _SCREAMING_SNAKE_CASE  			=					self.convert_tokens_to_ids(_lowercase   )
          _SCREAMING_SNAKE_CASE  			=					tgt_lang_id
          return inputs






       def 							UpperCamelCase					(				self							,				A__							,				A__ = "en_XX"							,				A__ = None							,				A__ = "ro_RO"							,				**A__							,				)     ->      BatchEncoding:
          _SCREAMING_SNAKE_CASE  			=					src_lang
          _SCREAMING_SNAKE_CASE  			=					tgt_lang
          return super().prepare_seqaseq_batch(_lowercase							,				_lowercase							,				**_lowercase   )






       def 							UpperCamelCase					(				self   )     ->      Dict:
          return self.set_src_lang_special_tokens(self.src_lang   )






       def 							UpperCamelCase					(				self   )     ->      Optional[int]:
          return self.set_tgt_lang_special_tokens(self.tgt_lang   )






       def 							UpperCamelCase					(				self							,				A__   )     ->      None:
          _SCREAMING_SNAKE_CASE  			=					self.convert_tokens_to_ids(_lowercase   )
          _SCREAMING_SNAKE_CASE  			=					[]
          _SCREAMING_SNAKE_CASE  			=					[self.eos_token_id, self.cur_lang_code]

          _SCREAMING_SNAKE_CASE  			=					self.convert_ids_to_tokens(self.prefix_tokens   )
          _SCREAMING_SNAKE_CASE  			=					self.convert_ids_to_tokens(self.suffix_tokens   )

          _SCREAMING_SNAKE_CASE  			=					processors.TemplateProcessing(
              single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str							,				pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str							,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,				self.prefix_tokens + self.suffix_tokens   )   )							,				)






       def 							UpperCamelCase					(				self							,				A__   )     ->      None:
          _SCREAMING_SNAKE_CASE  			=					self.convert_tokens_to_ids(_lowercase   )
          _SCREAMING_SNAKE_CASE  			=					[]
          _SCREAMING_SNAKE_CASE  			=					[self.eos_token_id, self.cur_lang_code]

          _SCREAMING_SNAKE_CASE  			=					self.convert_ids_to_tokens(self.prefix_tokens   )
          _SCREAMING_SNAKE_CASE  			=					self.convert_ids_to_tokens(self.suffix_tokens   )

          _SCREAMING_SNAKE_CASE  			=					processors.TemplateProcessing(
              single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str							,				pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str							,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,				self.prefix_tokens + self.suffix_tokens   )   )							,				)






       def 							UpperCamelCase					(				self							,				A__							,				A__ = None   )     ->      Tuple[str]:
          if not self.can_save_slow_tokenizer:
             raise ValueError(
                 """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                 """tokenizer."""   )

          if not os.path.isdir(_lowercase   ):
             logger.error(F"Vocabulary path ({save_directory}) should be a directory."   )
             return
          _SCREAMING_SNAKE_CASE  			=					os.path.join(
              _lowercase							,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )

          if os.path.abspath(self.vocab_file   ) != os.path.abspath(_lowercase   ):
             copyfile(self.vocab_file							,				_lowercase   )

          return (out_vocab_file,)









'''simple docstring'''

import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


SCREAMING_SNAKE_CASE							:				int     	=			get_tests_dir("fixtures")
SCREAMING_SNAKE_CASE							:				str     	=			get_tests_dir("fixtures/dummy_feature_extractor_config.json")
SCREAMING_SNAKE_CASE							:				str     	=			get_tests_dir("fixtures/dummy-config.json")






class    snake_case     (      unittest.TestCase		):






				"""simple docstring"""



				def   a__					(    self       )     ->							str:
											SCREAMING_SNAKE_CASE_				=				0



				def   a__					(    self       )     ->							Tuple:
											SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained('facebook/wav2vec2-base-960h'       )
											self.assertIsInstance(_lowercase,     _lowercase       )



				def   a__					(    self       )     ->							List[str]:
											SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained(_lowercase       )
											self.assertIsInstance(_lowercase,     _lowercase       )



				def   a__					(    self       )     ->							str:
											with tempfile.TemporaryDirectory() as tmpdirname:
																		SCREAMING_SNAKE_CASE_				=				WavaVecaConfig()

																		# remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
																		SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained(_lowercase       ).to_dict()

																		config_dict.pop('feature_extractor_type'       )
																		SCREAMING_SNAKE_CASE_				=				WavaVecaFeatureExtractor(**_lowercase       )

																		# save in new folder
																		model_config.save_pretrained(_lowercase       )
																		config.save_pretrained(_lowercase       )

																		SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained(_lowercase       )

																		# make sure private variable is not incorrectly saved
																		SCREAMING_SNAKE_CASE_				=				json.loads(config.to_json_string()       )
																		self.assertTrue('_processor_class' not in dict_as_saved       )

											self.assertIsInstance(_lowercase,     _lowercase       )



				def   a__					(    self       )     ->							int:
											SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained(_lowercase       )
											self.assertIsInstance(_lowercase,     _lowercase       )



				def   a__					(    self       )     ->							Any:
											with self.assertRaisesRegex(
											    _lowercase,     'bert-base is not a local folder and is not a valid model identifier'       ):
																		SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained('bert-base'       )



				def   a__					(    self       )     ->							List[Any]:
											with self.assertRaisesRegex(
											    _lowercase,     R'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'       ):
																		SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained(_lowercase,     revision='aaaaaa'       )



				def   a__					(    self       )     ->							List[Any]:
											with self.assertRaisesRegex(
											    _lowercase,     'hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.',     ):
																		SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained('hf-internal-testing/config-no-model'       )



				def   a__					(    self       )     ->							List[str]:
											# If remote code is not set, we will time out when asking whether to load the model.
											with self.assertRaises(_lowercase       ):
																		SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained(
																		    'hf-internal-testing/test_dynamic_feature_extractor'       )
											# If remote code is disabled, we can't load this config.
											with self.assertRaises(_lowercase       ):
																		SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained(
																		    'hf-internal-testing/test_dynamic_feature_extractor',     trust_remote_code=_lowercase       )

											SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained(
											    'hf-internal-testing/test_dynamic_feature_extractor',     trust_remote_code=_lowercase       )
											self.assertEqual(feature_extractor.__class__.__name__,     'NewFeatureExtractor'       )

											# Test feature extractor can be reloaded.
											with tempfile.TemporaryDirectory() as tmp_dir:
																		feature_extractor.save_pretrained(_lowercase       )
																		SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained(_lowercase,     trust_remote_code=_lowercase       )
											self.assertEqual(reloaded_feature_extractor.__class__.__name__,     'NewFeatureExtractor'       )



				def   a__					(    self       )     ->							int:
											try:
																		AutoConfig.register('custom',     _lowercase       )
																		AutoFeatureExtractor.register(_lowercase,     _lowercase       )
																		# Trying to register something existing in the Transformers library will raise an error
																		with self.assertRaises(_lowercase       ):
																									AutoFeatureExtractor.register(_lowercase,     _lowercase       )

																		# Now that the config is registered, it can be used as any other config with the auto-API
																		SCREAMING_SNAKE_CASE_				=				CustomFeatureExtractor.from_pretrained(_lowercase       )
																		with tempfile.TemporaryDirectory() as tmp_dir:
																									feature_extractor.save_pretrained(_lowercase       )
																									SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained(_lowercase       )
																									self.assertIsInstance(_lowercase,     _lowercase       )

											finally:
																		if "custom" in CONFIG_MAPPING._extra_content:
																									del CONFIG_MAPPING._extra_content["custom"]
																		if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																									del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]





				def   a__					(    self       )     ->							str:






											class    snake_case     (      lowercase_		):






															"""simple docstring"""
															_a    	=     True

											try:
																		AutoConfig.register('custom',     _lowercase       )
																		AutoFeatureExtractor.register(_lowercase,     _lowercase       )
																		# If remote code is not set, the default is to use local
																		SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained(
																		    'hf-internal-testing/test_dynamic_feature_extractor'       )
																		self.assertEqual(feature_extractor.__class__.__name__,     'NewFeatureExtractor'       )
																		self.assertTrue(feature_extractor.is_local       )

																		# If remote code is disabled, we load the local one.
																		SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained(
																		    'hf-internal-testing/test_dynamic_feature_extractor',     trust_remote_code=_lowercase       )
																		self.assertEqual(feature_extractor.__class__.__name__,     'NewFeatureExtractor'       )
																		self.assertTrue(feature_extractor.is_local       )

																		# If remote is enabled, we load from the Hub
																		SCREAMING_SNAKE_CASE_				=				AutoFeatureExtractor.from_pretrained(
																		    'hf-internal-testing/test_dynamic_feature_extractor',     trust_remote_code=_lowercase       )
																		self.assertEqual(feature_extractor.__class__.__name__,     'NewFeatureExtractor'       )
																		self.assertTrue(not hasattr(_lowercase,     'is_local'       )       )

											finally:
																		if "custom" in CONFIG_MAPPING._extra_content:
																									del CONFIG_MAPPING._extra_content["custom"]
																		if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																									del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]





