






import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


a__     :   Any       =     logging.get_logger(__name__)

a__     :   Union[str, Any]       =     """▁"""

a__     :   Any       =     {
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

a__     :   Union[str, Any]       =     {
    """vocab_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json""",
    },
    """spm_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_config_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json""",
    },
}

a__     :   Tuple       =     {
    """facebook/m2m100_418M""": 1_0_2_4,
}

# fmt: off
a__     :   Optional[Any]       =     {
    """m2m100""": ["""af""", """am""", """ar""", """ast""", """az""", """ba""", """be""", """bg""", """bn""", """br""", """bs""", """ca""", """ceb""", """cs""", """cy""", """da""", """de""", """el""", """en""", """es""", """et""", """fa""", """ff""", """fi""", """fr""", """fy""", """ga""", """gd""", """gl""", """gu""", """ha""", """he""", """hi""", """hr""", """ht""", """hu""", """hy""", """id""", """ig""", """ilo""", """is""", """it""", """ja""", """jv""", """ka""", """kk""", """km""", """kn""", """ko""", """lb""", """lg""", """ln""", """lo""", """lt""", """lv""", """mg""", """mk""", """ml""", """mn""", """mr""", """ms""", """my""", """ne""", """nl""", """no""", """ns""", """oc""", """or""", """pa""", """pl""", """ps""", """pt""", """ro""", """ru""", """sd""", """si""", """sk""", """sl""", """so""", """sq""", """sr""", """ss""", """su""", """sv""", """sw""", """ta""", """th""", """tl""", """tn""", """tr""", """uk""", """ur""", """uz""", """vi""", """wo""", """xh""", """yi""", """yo""", """zh""", """zu"""],
    """wmt21""": ["""en""", """ha""", """is""", """ja""", """cs""", """ru""", """zh""", """de"""]
}


class    _lowerCAmelCase			(		lowercase_      ):



				"""simple docstring"""



				_lowercase				:   Dict      =  VOCAB_FILES_NAMES
				_lowercase				:   Any      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_lowercase				:   Optional[int]      =  PRETRAINED_VOCAB_FILES_MAP
				_lowercase				:   Dict      =  ['''input_ids''', '''attention_mask''']

				_lowercase				:   List[int]      =  []
				_lowercase				:   List[int]      =  []



				def __init__(      self    :    int			,		UpperCamelCase__    :    List[Any]			,		UpperCamelCase__    :    Tuple			,		UpperCamelCase__    :    Union[str, Any]=None			,		UpperCamelCase__    :    int=None			,		UpperCamelCase__    :    int="<s>"			,		UpperCamelCase__    :    List[str]="</s>"			,		UpperCamelCase__    :    Optional[Any]="</s>"			,		UpperCamelCase__    :    Dict="<pad>"			,		UpperCamelCase__    :    Union[str, Any]="<unk>"			,		UpperCamelCase__    :    Dict="m2m100"			,		UpperCamelCase__    :    Optional[Dict[str, Any]] = None			,		UpperCamelCase__    :    Optional[Any]=8			,		**UpperCamelCase__    :    Any			,		):






											'''simple docstring'''







											snake_case__	  =       {} if sp_model_kwargs is None else sp_model_kwargs

											snake_case__	  =       language_codes
											snake_case__	  =       FAIRSEQ_LANGUAGE_CODES[language_codes]
											snake_case__	  =       {lang_code: F'''__{lang_code}__''' for lang_code in fairseq_language_code}

											snake_case__	  =       kwargs.get("""additional_special_tokens"""			,		[])
											kwargs["additional_special_tokens"] += [
											    self.get_lang_token(UpperCamelCase__)
											    for lang_code in fairseq_language_code
											    if self.get_lang_token(UpperCamelCase__) not in kwargs["additional_special_tokens"]
											]

											super().__init__(
											    src_lang=UpperCamelCase__			,		tgt_lang=UpperCamelCase__			,		bos_token=UpperCamelCase__			,		eos_token=UpperCamelCase__			,		sep_token=UpperCamelCase__			,		unk_token=UpperCamelCase__			,		pad_token=UpperCamelCase__			,		language_codes=UpperCamelCase__			,		sp_model_kwargs=self.sp_model_kwargs			,		num_madeup_words=UpperCamelCase__			,		**UpperCamelCase__			,		)

											snake_case__	  =       vocab_file
											snake_case__	  =       load_json(UpperCamelCase__)
											snake_case__	  =       {v: k for k, v in self.encoder.items()}
											snake_case__	  =       spm_file
											snake_case__	  =       load_spm(UpperCamelCase__			,		self.sp_model_kwargs)

											snake_case__	  =       len(self.encoder)

											snake_case__	  =       {
											    self.get_lang_token(UpperCamelCase__): self.encoder_size + i for i, lang_code in enumerate(UpperCamelCase__)
											}
											snake_case__	  =       {lang_code: self.encoder_size + i for i, lang_code in enumerate(UpperCamelCase__)}
											snake_case__	  =       {v: k for k, v in self.lang_token_to_id.items()}

											snake_case__	  =       src_lang if src_lang is not None else """en"""
											snake_case__	  =       tgt_lang
											snake_case__	  =       self.get_lang_id(self._src_lang)
											self.set_src_lang_special_tokens(self._src_lang)

											snake_case__	  =       num_madeup_words



				@property
				def 					__magic_name__							(      self    :    int):






											'''simple docstring'''







											return len(self.encoder) + len(self.lang_token_to_id)



				@property
				def 					__magic_name__							(      self    :    Dict):






											'''simple docstring'''







											return self._src_lang



				@src_lang.setter
				def 					__magic_name__							(      self    :    List[Any]			,		UpperCamelCase__    :    str):






											'''simple docstring'''







											snake_case__	  =       new_src_lang
											self.set_src_lang_special_tokens(self._src_lang)



				def 					__magic_name__							(      self    :    Union[str, Any]			,		UpperCamelCase__    :    str):






											'''simple docstring'''







											return self.sp_model.encode(UpperCamelCase__			,		out_type=UpperCamelCase__)



				def 					__magic_name__							(      self    :    Dict			,		UpperCamelCase__    :    Optional[Any]):






											'''simple docstring'''







											if token in self.lang_token_to_id:
																		return self.lang_token_to_id[token]
											return self.encoder.get(UpperCamelCase__			,		self.encoder[self.unk_token])



				def 					__magic_name__							(      self    :    Dict			,		UpperCamelCase__    :    int):






											'''simple docstring'''







											if index in self.id_to_lang_token:
																		return self.id_to_lang_token[index]
											return self.decoder.get(UpperCamelCase__			,		self.unk_token)



				def 					__magic_name__							(      self    :    Union[str, Any]			,		UpperCamelCase__    :    Union[str, Any]):






											'''simple docstring'''







											snake_case__	  =       []
											snake_case__	  =       """"""
											for token in tokens:
																		# make sure that special tokens are not decoded using sentencepiece model
																		if token in self.all_special_tokens:
																									out_string += self.sp_model.decode(UpperCamelCase__) + token
																									snake_case__	  =       []
																		else:
																									current_sub_tokens.append(UpperCamelCase__)
											out_string += self.sp_model.decode(UpperCamelCase__)
											return out_string.strip()



				def 					__magic_name__							(      self    :    int			,		UpperCamelCase__    :    List[int]			,		UpperCamelCase__    :    Optional[List[int]] = None			,		UpperCamelCase__    :    bool = False):






											'''simple docstring'''







											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=UpperCamelCase__			,		token_ids_a=UpperCamelCase__			,		already_has_special_tokens=UpperCamelCase__)

											snake_case__	  =       [1] * len(self.prefix_tokens)
											snake_case__	  =       [1] * len(self.suffix_tokens)
											if token_ids_a is None:
																		return prefix_ones + ([0] * len(UpperCamelCase__)) + suffix_ones
											return prefix_ones + ([0] * len(UpperCamelCase__)) + ([0] * len(UpperCamelCase__)) + suffix_ones



				def 					__magic_name__							(      self    :    List[str]			,		UpperCamelCase__    :    List[int]			,		UpperCamelCase__    :    Optional[List[int]] = None):






											'''simple docstring'''







											if token_ids_a is None:
																		return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



				def 					__magic_name__							(      self    :    int):






											'''simple docstring'''







											snake_case__	  =       {self.convert_ids_to_tokens(UpperCamelCase__): i for i in range(self.vocab_size)}
											vocab.update(self.added_tokens_encoder)
											return vocab



				def __getstate__(      self    :    str):






											'''simple docstring'''







											snake_case__	  =       self.__dict__.copy()
											snake_case__	  =       None
											return state



				def __setstate__(      self    :    str			,		UpperCamelCase__    :    Dict):






											'''simple docstring'''







											snake_case__	  =       d

											# for backward compatibility
											if not hasattr(self			,		"""sp_model_kwargs"""):
																		snake_case__	  =       {}

											snake_case__	  =       load_spm(self.spm_file			,		self.sp_model_kwargs)



				def 					__magic_name__							(      self    :    Tuple			,		UpperCamelCase__    :    str			,		UpperCamelCase__    :    Optional[str] = None):






											'''simple docstring'''







											snake_case__	  =       Path(UpperCamelCase__)
											if not save_dir.is_dir():
																		raise OSError(F'''{save_directory} should be a directory''')
											snake_case__	  =       save_dir / (
											    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
											)
											snake_case__	  =       save_dir / (
											    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
											)

											save_json(self.encoder			,		UpperCamelCase__)

											if os.path.abspath(self.spm_file) != os.path.abspath(UpperCamelCase__) and os.path.isfile(self.spm_file):
																		copyfile(self.spm_file			,		UpperCamelCase__)
											elif not os.path.isfile(self.spm_file):
																		with open(UpperCamelCase__			,		"""wb""") as fi:
																									snake_case__	  =       self.sp_model.serialized_model_proto()
																									fi.write(UpperCamelCase__)

											return (str(UpperCamelCase__), str(UpperCamelCase__))



				def 					__magic_name__							(      self    :    List[Any]			,		UpperCamelCase__    :    List[str]			,		UpperCamelCase__    :    str = "en"			,		UpperCamelCase__    :    Optional[List[str]] = None			,		UpperCamelCase__    :    str = "ro"			,		**UpperCamelCase__    :    Tuple			,		):






											'''simple docstring'''







											snake_case__	  =       src_lang
											snake_case__	  =       tgt_lang
											self.set_src_lang_special_tokens(self.src_lang)
											return super().prepare_seqaseq_batch(UpperCamelCase__			,		UpperCamelCase__			,		**UpperCamelCase__)



				def 					__magic_name__							(      self    :    Optional[Any]			,		UpperCamelCase__    :    Optional[Any]			,		UpperCamelCase__    :    Optional[str]			,		UpperCamelCase__    :    Optional[str]			,		**UpperCamelCase__    :    Optional[int]):






											'''simple docstring'''







											if src_lang is None or tgt_lang is None:
																		raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model""")
											snake_case__	  =       src_lang
											snake_case__	  =       self(UpperCamelCase__			,		add_special_tokens=UpperCamelCase__			,		**UpperCamelCase__)
											snake_case__	  =       self.get_lang_id(UpperCamelCase__)
											snake_case__	  =       tgt_lang_id
											return inputs



				def 					__magic_name__							(      self    :    Optional[Any]):






											'''simple docstring'''







											self.set_src_lang_special_tokens(self.src_lang)



				def 					__magic_name__							(      self    :    Optional[int]):






											'''simple docstring'''







											self.set_tgt_lang_special_tokens(self.tgt_lang)



				def 					__magic_name__							(      self    :    str			,		UpperCamelCase__    :    str):






											'''simple docstring'''







											snake_case__	  =       self.get_lang_token(UpperCamelCase__)
											snake_case__	  =       self.lang_token_to_id[lang_token]
											snake_case__	  =       [self.cur_lang_id]
											snake_case__	  =       [self.eos_token_id]



				def 					__magic_name__							(      self    :    int			,		UpperCamelCase__    :    str):






											'''simple docstring'''







											snake_case__	  =       self.get_lang_token(UpperCamelCase__)
											snake_case__	  =       self.lang_token_to_id[lang_token]
											snake_case__	  =       [self.cur_lang_id]
											snake_case__	  =       [self.eos_token_id]



				def 					__magic_name__							(      self    :    Any			,		UpperCamelCase__    :    str):






											'''simple docstring'''







											return self.lang_code_to_token[lang]



				def 					__magic_name__							(      self    :    Any			,		UpperCamelCase__    :    str):






											'''simple docstring'''







											snake_case__	  =       self.get_lang_token(UpperCamelCase__)
											return self.lang_token_to_id[lang_token]


def _UpperCAmelCase   (	a						:	str  ,					a						:	Dict[str, Any] ):
							snake_case__	  =       sentencepiece.SentencePieceProcessor(**a )
							spm.Load(str(a ) )
							return spm

def _UpperCAmelCase   (	a						:	str ):
							with open(a  ,					"""r""" ) as f:
														return json.load(a )



def _UpperCAmelCase   (	a						:	Dict  ,					a						:	str ):
							with open(a  ,					"""w""" ) as f:
														json.dump(a  ,					a  ,					indent=2 )








import unittest

import torch

from diffusers import VQModel
from diffusers.utils import floats_tensor, torch_device
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


enable_full_determinism()


class    _lowerCAmelCase			(		lowercase_     ,      lowercase_     ,      unittest.TestCase      ):



				"""simple docstring"""



				_lowercase				:   Optional[int]      =  VQModel
				_lowercase				:   str      =  '''sample'''



				@property
				def 					__magic_name__							(      self    :    Union[str, Any]			,		UpperCamelCase__    :    Tuple=(3_2, 3_2)):






											'''simple docstring'''







											snake_case__	  =       4
											snake_case__	  =       3

											snake_case__	  =       floats_tensor((batch_size, num_channels) + sizes).to(UpperCamelCase__)

											return {"sample": image}



				@property
				def 					__magic_name__							(      self    :    str):






											'''simple docstring'''







											return (3, 3_2, 3_2)



				@property
				def 					__magic_name__							(      self    :    List[str]):






											'''simple docstring'''







											return (3, 3_2, 3_2)



				def 					__magic_name__							(      self    :    int):






											'''simple docstring'''







											snake_case__	  =       {
											    """block_out_channels""": [3_2, 6_4],
											    """in_channels""": 3,
											    """out_channels""": 3,
											    """down_block_types""": ["""DownEncoderBlock2D""", """DownEncoderBlock2D"""],
											    """up_block_types""": ["""UpDecoderBlock2D""", """UpDecoderBlock2D"""],
											    """latent_channels""": 3,
											}
											snake_case__	  =       self.dummy_input
											return init_dict, inputs_dict



				def 					__magic_name__							(      self    :    Optional[int]):






											'''simple docstring'''







											pass



				def 					__magic_name__							(      self    :    Tuple):






											'''simple docstring'''







											pass



				def 					__magic_name__							(      self    :    str):






											'''simple docstring'''







											snake_case__  , snake_case__	  =       VQModel.from_pretrained("""fusing/vqgan-dummy"""			,		output_loading_info=UpperCamelCase__)
											self.assertIsNotNone(UpperCamelCase__)
											self.assertEqual(len(loading_info["""missing_keys"""])			,		0)

											model.to(UpperCamelCase__)
											snake_case__	  =       model(**self.dummy_input)

											assert image is not None, "Make sure output is not None"



				def 					__magic_name__							(      self    :    Dict):






											'''simple docstring'''







											snake_case__	  =       VQModel.from_pretrained("""fusing/vqgan-dummy""")
											model.to(UpperCamelCase__).eval()

											torch.manual_seed(0)
											if torch.cuda.is_available():
																		torch.cuda.manual_seed_all(0)

											snake_case__	  =       torch.randn(1			,		model.config.in_channels			,		model.config.sample_size			,		model.config.sample_size)
											snake_case__	  =       image.to(UpperCamelCase__)
											with torch.no_grad():
																		snake_case__	  =       model(UpperCamelCase__).sample

											snake_case__	  =       output[0, -1, -3:, -3:].flatten().cpu()
											# fmt: off
											snake_case__	  =       torch.tensor([-0.01_53, -0.40_44, -0.18_80, -0.51_61, -0.24_18, -0.40_72, -0.16_12, -0.06_33, -0.01_43])
											# fmt: on
											self.assertTrue(torch.allclose(UpperCamelCase__			,		UpperCamelCase__			,		atol=1E-3))
