



'''simple docstring'''





import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


__a  			=						get_tests_dir("fixtures/test_sentencepiece.model")
__a  			=						get_tests_dir("fixtures/test_sentencepiece_bpe.model")

__a  			=						"pt" if is_torch_available() else "tf"







@require_sentencepiece
@require_tokenizers
class        UpperCAmelCase_			(			_a						,						unittest.TestCase      ):
    """simple docstring"""







    lowercase   =						CamembertTokenizer
    lowercase   =						CamembertTokenizerFast
    lowercase   =						True
    lowercase   =						True




    def lowerCamelCase			(       self   :	Tuple		):
          super().setUp()

          # We have a SentencePiece fixture for testing
          snake_case__    :       List[str]       		= CamembertTokenizer(snake_case_		)
          tokenizer.save_pretrained(self.tmpdirname		)




    def lowerCamelCase			(       self   :	int		):
          snake_case__    :       Optional[Any]       		= """<pad>"""
          snake_case__    :       List[Any]       		= 1

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case_		)					,	snake_case_		)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case_		)					,	snake_case_		)




    def lowerCamelCase			(       self   :	Union[str, Any]		):
          snake_case__    :       str       		= list(self.get_tokenizer().get_vocab().keys()		)

          self.assertEqual(vocab_keys[0]					,	"""<s>NOTUSED"""		)
          self.assertEqual(vocab_keys[1]					,	"""<pad>"""		)
          self.assertEqual(vocab_keys[-1]					,	"""<mask>"""		)
          self.assertEqual(len(snake_case_		)					,	1_004		)




    def lowerCamelCase			(       self   :	Tuple		):
          self.assertEqual(self.get_tokenizer().vocab_size					,	1_005		)




    def lowerCamelCase			(       self   :	Optional[Any]		):
          snake_case__    :       Dict       		= CamembertTokenizer(snake_case_		)
          tokenizer.save_pretrained(self.tmpdirname		)
          snake_case__    :       Dict       		= CamembertTokenizerFast.from_pretrained(self.tmpdirname		)

          snake_case__    :       Tuple       		= """I was born in 92000, and this is falsé."""

          snake_case__    :       Optional[Any]       		= tokenizer.encode(snake_case_		)
          snake_case__    :       List[Any]       		= rust_tokenizer.encode(snake_case_		)
          self.assertListEqual(snake_case_					,	snake_case_		)

          snake_case__    :       Any       		= tokenizer.encode(snake_case_					,	add_special_tokens=snake_case_		)
          snake_case__    :       Optional[int]       		= rust_tokenizer.encode(snake_case_					,	add_special_tokens=snake_case_		)
          self.assertListEqual(snake_case_					,	snake_case_		)

          # <unk> tokens are not the same for `rust` than for `slow`.
          # Because spm gives back raw token instead of `unk` in EncodeAsPieces
          # tokens = tokenizer.tokenize(sequence)
          snake_case__    :       Optional[int]       		= tokenizer.convert_ids_to_tokens(snake_case_		)
          snake_case__    :       List[Any]       		= rust_tokenizer.tokenize(snake_case_		)
          self.assertListEqual(snake_case_					,	snake_case_		)




    def lowerCamelCase			(       self   :	Any		):
          if not self.test_rust_tokenizer:
                return

          snake_case__    :       Dict       		= self.get_tokenizer()
          snake_case__    :       List[Any]       		= self.get_rust_tokenizer()

          snake_case__    :       str       		= """I was born in 92000, and this is falsé."""

          snake_case__    :       Optional[Any]       		= tokenizer.tokenize(snake_case_		)
          snake_case__    :       Union[str, Any]       		= rust_tokenizer.tokenize(snake_case_		)
          self.assertListEqual(snake_case_					,	snake_case_		)

          snake_case__    :       int       		= tokenizer.encode(snake_case_					,	add_special_tokens=snake_case_		)
          snake_case__    :       Tuple       		= rust_tokenizer.encode(snake_case_					,	add_special_tokens=snake_case_		)
          self.assertListEqual(snake_case_					,	snake_case_		)

          snake_case__    :       Union[str, Any]       		= self.get_rust_tokenizer()
          snake_case__    :       str       		= tokenizer.encode(snake_case_		)
          snake_case__    :       List[Any]       		= rust_tokenizer.encode(snake_case_		)
          self.assertListEqual(snake_case_					,	snake_case_		)




    @slow
    def lowerCamelCase			(       self   :	Optional[Any]		):
          # fmt: off
          snake_case__    :       Tuple       		= {"""input_ids""": [[5, 54, 7_196, 297, 30, 23, 776, 18, 11, 3_215, 3_705, 8_252, 22, 3_164, 1_181, 2_116, 29, 16, 813, 25, 791, 3_314, 20, 3_446, 38, 27_575, 120, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 468, 17, 11, 9_088, 20, 1_517, 8, 22_804, 18_818, 10, 38, 629, 607, 607, 142, 19, 7_196, 867, 56, 10_326, 24, 2_267, 20, 416, 5_072, 15_612, 233, 734, 7, 2_399, 27, 16, 3_015, 1_649, 7, 24, 20, 4_338, 2_399, 27, 13, 3_400, 14, 13, 6_189, 8, 930, 9, 6]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
          # fmt: on

          # camembert is a french model. So we also use french texts.
          snake_case__    :       Optional[int]       		= [
              """Le transformeur est un modèle d'apprentissage profond introduit en 2017, """
              """utilisé principalement dans le domaine du traitement automatique des langues (TAL).""",
              """À l'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus """
              """pour gérer des données séquentielles, telles que le langage naturel, pour des tâches """
              """telles que la traduction et la synthèse de texte.""",
          ]

          self.tokenizer_integration_test_util(
              expected_encoding=snake_case_					,	model_name="""camembert-base"""					,	revision="""3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf"""					,	sequences=snake_case_					,	)





'''simple docstring'''





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
       from .tokenization_nllb import NllbTokenizer
else:
       __a  			=						None


__a  			=						logging.get_logger(__name__)


__a  			=						{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

__a  			=						{
    "vocab_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json"
        ),
    },
}

__a  			=						{
    "facebook/nllb-large-en-ro": 1024,
    "facebook/nllb-200-distilled-600M": 1024,
}

# fmt: off
__a  			=						["ace_Arab", "ace_Latn", "acm_Arab", "acq_Arab", "aeb_Arab", "afr_Latn", "ajp_Arab", "aka_Latn", "amh_Ethi", "apc_Arab", "arb_Arab", "ars_Arab", "ary_Arab", "arz_Arab", "asm_Beng", "ast_Latn", "awa_Deva", "ayr_Latn", "azb_Arab", "azj_Latn", "bak_Cyrl", "bam_Latn", "ban_Latn", "bel_Cyrl", "bem_Latn", "ben_Beng", "bho_Deva", "bjn_Arab", "bjn_Latn", "bod_Tibt", "bos_Latn", "bug_Latn", "bul_Cyrl", "cat_Latn", "ceb_Latn", "ces_Latn", "cjk_Latn", "ckb_Arab", "crh_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "dik_Latn", "dyu_Latn", "dzo_Tibt", "ell_Grek", "eng_Latn", "epo_Latn", "est_Latn", "eus_Latn", "ewe_Latn", "fao_Latn", "pes_Arab", "fij_Latn", "fin_Latn", "fon_Latn", "fra_Latn", "fur_Latn", "fuv_Latn", "gla_Latn", "gle_Latn", "glg_Latn", "grn_Latn", "guj_Gujr", "hat_Latn", "hau_Latn", "heb_Hebr", "hin_Deva", "hne_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ibo_Latn", "ilo_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jav_Latn", "jpn_Jpan", "kab_Latn", "kac_Latn", "kam_Latn", "kan_Knda", "kas_Arab", "kas_Deva", "kat_Geor", "knc_Arab", "knc_Latn", "kaz_Cyrl", "kbp_Latn", "kea_Latn", "khm_Khmr", "kik_Latn", "kin_Latn", "kir_Cyrl", "kmb_Latn", "kon_Latn", "kor_Hang", "kmr_Latn", "lao_Laoo", "lvs_Latn", "lij_Latn", "lim_Latn", "lin_Latn", "lit_Latn", "lmo_Latn", "ltg_Latn", "ltz_Latn", "lua_Latn", "lug_Latn", "luo_Latn", "lus_Latn", "mag_Deva", "mai_Deva", "mal_Mlym", "mar_Deva", "min_Latn", "mkd_Cyrl", "plt_Latn", "mlt_Latn", "mni_Beng", "khk_Cyrl", "mos_Latn", "mri_Latn", "zsm_Latn", "mya_Mymr", "nld_Latn", "nno_Latn", "nob_Latn", "npi_Deva", "nso_Latn", "nus_Latn", "nya_Latn", "oci_Latn", "gaz_Latn", "ory_Orya", "pag_Latn", "pan_Guru", "pap_Latn", "pol_Latn", "por_Latn", "prs_Arab", "pbt_Arab", "quy_Latn", "ron_Latn", "run_Latn", "rus_Cyrl", "sag_Latn", "san_Deva", "sat_Beng", "scn_Latn", "shn_Mymr", "sin_Sinh", "slk_Latn", "slv_Latn", "smo_Latn", "sna_Latn", "snd_Arab", "som_Latn", "sot_Latn", "spa_Latn", "als_Latn", "srd_Latn", "srp_Cyrl", "ssw_Latn", "sun_Latn", "swe_Latn", "swh_Latn", "szl_Latn", "tam_Taml", "tat_Cyrl", "tel_Telu", "tgk_Cyrl", "tgl_Latn", "tha_Thai", "tir_Ethi", "taq_Latn", "taq_Tfng", "tpi_Latn", "tsn_Latn", "tso_Latn", "tuk_Latn", "tum_Latn", "tur_Latn", "twi_Latn", "tzm_Tfng", "uig_Arab", "ukr_Cyrl", "umb_Latn", "urd_Arab", "uzn_Latn", "vec_Latn", "vie_Latn", "war_Latn", "wol_Latn", "xho_Latn", "ydd_Hebr", "yor_Latn", "yue_Hant", "zho_Hans", "zho_Hant", "zul_Latn"]







class        UpperCAmelCase_			(			_a      ):
    """simple docstring"""







    lowercase   =						VOCAB_FILES_NAMES
    lowercase   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase   =						PRETRAINED_VOCAB_FILES_MAP
    lowercase   =						["input_ids", "attention_mask"]
    lowercase   =						NllbTokenizer

    lowercase   =						[]
    lowercase   =						[]




    def __init__(       self   :	List[str]					,	snake_case_   :	int=None					,	snake_case_   :	Optional[int]=None					,	snake_case_   :	Dict="<s>"					,	snake_case_   :	Optional[Any]="</s>"					,	snake_case_   :	Union[str, Any]="</s>"					,	snake_case_   :	Optional[int]="<s>"					,	snake_case_   :	Any="<unk>"					,	snake_case_   :	Tuple="<pad>"					,	snake_case_   :	Any="<mask>"					,	snake_case_   :	Union[str, Any]=None					,	snake_case_   :	Tuple=None					,	snake_case_   :	Union[str, Any]=None					,	snake_case_   :	Optional[int]=False					,	**snake_case_   :	List[Any]					,	):
          # Mask token behave like a normal word, i.e. include the space before it
          snake_case__    :       Any       		= AddedToken(snake_case_					,	lstrip=snake_case_					,	rstrip=snake_case_		) if isinstance(snake_case_					,	snake_case_		) else mask_token
          snake_case__    :       str       		= legacy_behaviour
          super().__init__(
              vocab_file=snake_case_					,	tokenizer_file=snake_case_					,	bos_token=snake_case_					,	eos_token=snake_case_					,	sep_token=snake_case_					,	cls_token=snake_case_					,	unk_token=snake_case_					,	pad_token=snake_case_					,	mask_token=snake_case_					,	src_lang=snake_case_					,	tgt_lang=snake_case_					,	additional_special_tokens=snake_case_					,	legacy_behaviour=snake_case_					,	**snake_case_					,	)

          snake_case__    :       Optional[Any]       		= vocab_file
          snake_case__    :       str       		= False if not self.vocab_file else True

          snake_case__    :       List[str]       		= FAIRSEQ_LANGUAGE_CODES.copy()

          if additional_special_tokens is not None:
                # Only add those special tokens if they are not already there.
                _additional_special_tokens.extend(
                    [t for t in additional_special_tokens if t not in _additional_special_tokens]		)

          self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}		)
          snake_case__    :       Optional[int]       		= {
              lang_code: self.convert_tokens_to_ids(snake_case_		) for lang_code in FAIRSEQ_LANGUAGE_CODES
          }

          snake_case__    :       Any       		= src_lang if src_lang is not None else """eng_Latn"""
          snake_case__    :       Optional[int]       		= self.convert_tokens_to_ids(self._src_lang		)
          snake_case__    :       Any       		= tgt_lang
          self.set_src_lang_special_tokens(self._src_lang		)




    @property
    def lowerCamelCase			(       self   :	str		):
          return self._src_lang




    @src_lang.setter
    def lowerCamelCase			(       self   :	str					,	snake_case_   :	str		):
          snake_case__    :       Any       		= new_src_lang
          self.set_src_lang_special_tokens(self._src_lang		)




    def lowerCamelCase			(       self   :	int					,	snake_case_   :	List[int]					,	snake_case_   :	Optional[List[int]] = None		):
          if token_ids_a is None:
                return self.prefix_tokens + token_ids_a + self.suffix_tokens
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




    def lowerCamelCase			(       self   :	Any					,	snake_case_   :	List[int]					,	snake_case_   :	Optional[List[int]] = None		):
          snake_case__    :       int       		= [self.sep_token_id]
          snake_case__    :       Optional[int]       		= [self.cls_token_id]

          if token_ids_a is None:
                return len(cls + token_ids_a + sep		) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]




    def lowerCamelCase			(       self   :	List[str]					,	snake_case_   :	str					,	snake_case_   :	str					,	snake_case_   :	Optional[str]					,	snake_case_   :	Optional[str]					,	**snake_case_   :	List[Any]		):
          if src_lang is None or tgt_lang is None:
                raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""		)
          snake_case__    :       Any       		= src_lang
          snake_case__    :       str       		= self(snake_case_					,	add_special_tokens=snake_case_					,	return_tensors=snake_case_					,	**snake_case_		)
          snake_case__    :       Dict       		= self.convert_tokens_to_ids(snake_case_		)
          snake_case__    :       str       		= tgt_lang_id
          return inputs




    def lowerCamelCase			(       self   :	int					,	snake_case_   :	List[str]					,	snake_case_   :	str = "eng_Latn"					,	snake_case_   :	Optional[List[str]] = None					,	snake_case_   :	str = "fra_Latn"					,	**snake_case_   :	str					,	):
          snake_case__    :       str       		= src_lang
          snake_case__    :       List[Any]       		= tgt_lang
          return super().prepare_seqaseq_batch(snake_case_					,	snake_case_					,	**snake_case_		)




    def lowerCamelCase			(       self   :	List[str]		):
          return self.set_src_lang_special_tokens(self.src_lang		)




    def lowerCamelCase			(       self   :	Any		):
          return self.set_tgt_lang_special_tokens(self.tgt_lang		)




    def lowerCamelCase			(       self   :	Optional[Any]					,	snake_case_   :	List[str]		):
          snake_case__    :       List[Any]       		= self.convert_tokens_to_ids(snake_case_		)

          if self.legacy_behaviour:
                snake_case__    :       Tuple       		= []
                snake_case__    :       Optional[int]       		= [self.eos_token_id, self.cur_lang_code]
          else:
                snake_case__    :       Tuple       		= [self.cur_lang_code]
                snake_case__    :       int       		= [self.eos_token_id]

          snake_case__    :       List[str]       		= self.convert_ids_to_tokens(self.prefix_tokens		)
          snake_case__    :       Tuple       		= self.convert_ids_to_tokens(self.suffix_tokens		)

          snake_case__    :       Union[str, Any]       		= processors.TemplateProcessing(
              single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str					,	pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str					,	special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str					,	self.prefix_tokens + self.suffix_tokens		)		)					,	)




    def lowerCamelCase			(       self   :	int					,	snake_case_   :	str		):
          snake_case__    :       List[str]       		= self.convert_tokens_to_ids(snake_case_		)
          if self.legacy_behaviour:
                snake_case__    :       int       		= []
                snake_case__    :       Optional[Any]       		= [self.eos_token_id, self.cur_lang_code]
          else:
                snake_case__    :       Dict       		= [self.cur_lang_code]
                snake_case__    :       Dict       		= [self.eos_token_id]

          snake_case__    :       Dict       		= self.convert_ids_to_tokens(self.prefix_tokens		)
          snake_case__    :       Any       		= self.convert_ids_to_tokens(self.suffix_tokens		)

          snake_case__    :       List[Any]       		= processors.TemplateProcessing(
              single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str					,	pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str					,	special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str					,	self.prefix_tokens + self.suffix_tokens		)		)					,	)




    def lowerCamelCase			(       self   :	str					,	snake_case_   :	str					,	snake_case_   :	Optional[str] = None		):
          if not self.can_save_slow_tokenizer:
                raise ValueError(
                    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                    """tokenizer."""		)

          if not os.path.isdir(snake_case_		):
                logger.error(f"Vocabulary path ({save_directory}) should be a directory."		)
                return
          snake_case__    :       Dict       		= os.path.join(
              snake_case_					,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

          if os.path.abspath(self.vocab_file		) != os.path.abspath(snake_case_		):
                copyfile(self.vocab_file					,	snake_case_		)

          return (out_vocab_file,)
