





"""simple docstring"""


import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__magic_name__       :					Optional[int]						=			logging.get_logger(__name__)

__magic_name__       :					List[Any]						=			"""▁"""

__magic_name__       :					str						=			{
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
}

__magic_name__       :					Optional[Any]						=			{
    """vocab_file""": {
        """facebook/s2t-small-librispeech-asr""": (
            """https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json"""
        ),
    },
    """spm_file""": {
        """facebook/s2t-small-librispeech-asr""": (
            """https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model"""
        )
    },
}

__magic_name__       :					Union[str, Any]						=			{
    """facebook/s2t-small-librispeech-asr""": 1_0_2_4,
}

__magic_name__       :					Optional[int]						=			["""pt""", """fr""", """ru""", """nl""", """ro""", """it""", """es""", """de"""]

__magic_name__       :					Tuple						=			{"""mustc""": MUSTC_LANGS}






class 							lowercase__			(   __SCREAMING_SNAKE_CASE	):

			"""simple docstring"""
			__lowerCAmelCase     :				Any										=  VOCAB_FILES_NAMES
			__lowerCAmelCase     :				Tuple										=  PRETRAINED_VOCAB_FILES_MAP
			__lowerCAmelCase     :				List[Any]										=  MAX_MODEL_INPUT_SIZES
			__lowerCAmelCase     :				int										=  ["""input_ids""", """attention_mask"""]

			__lowerCAmelCase     :				List[int]										=  []







			def __init__(      self						,   _A						,   _A						,   _A="<s>"						,   _A="</s>"						,   _A="<pad>"						,   _A="<unk>"						,   _A=False						,   _A=False						,   _A=None						,   _A=None						,   _A = None						,   **_A						,   ):






									'''simple docstring'''







									UpperCamelCase				:		Union[str, Any]				   =			{} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    bos_token=_A						,   eos_token=_A						,   unk_token=_A						,   pad_token=_A						,   do_upper_case=_A						,   do_lower_case=_A						,   tgt_lang=_A						,   lang_codes=_A						,   sp_model_kwargs=self.sp_model_kwargs						,   **_A						,   )
									UpperCamelCase				:		Any				   =			do_upper_case
									UpperCamelCase				:		Tuple				   =			do_lower_case

									UpperCamelCase				:		Tuple				   =			load_json(_A    )
									UpperCamelCase				:		List[str]				   =			{v: k for k, v in self.encoder.items()}
									UpperCamelCase				:		Union[str, Any]				   =			spm_file
									UpperCamelCase				:		Any				   =			load_spm(_A						,   self.sp_model_kwargs    )

									if lang_codes is not None:
															UpperCamelCase				:		List[str]				   =			lang_codes
															UpperCamelCase				:		Tuple				   =			LANGUAGES[lang_codes]
															UpperCamelCase				:		List[str]				   =			[f"""<lang:{lang}>""" for lang in self.langs]
															UpperCamelCase				:		str				   =			{lang: self.sp_model.PieceToId(f"""<lang:{lang}>"""    ) for lang in self.langs}

															UpperCamelCase				:		Dict				   =			self.lang_tokens
															UpperCamelCase				:		Any				   =			tgt_lang if tgt_lang is not None else self.langs[0]

															self.set_tgt_lang_special_tokens(self._tgt_lang    )
									else:
															UpperCamelCase				:		List[Any]				   =			{}







			@property
			def   _a      (      self    ):






									'''simple docstring'''







									return len(self.encoder    )







			@property
			def   _a      (      self    ):






									'''simple docstring'''







									return self._tgt_lang







			@tgt_lang.setter
			def   _a      (      self						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		List[Any]				   =			new_tgt_lang
									self.set_tgt_lang_special_tokens(_A    )







			def   _a      (      self						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		Dict				   =			self.lang_code_to_id[tgt_lang]
									UpperCamelCase				:		int				   =			[lang_code_id]







			def   _a      (      self						,   _A    ):






									'''simple docstring'''







									return self.sp_model.encode(_A						,   out_type=_A    )







			def   _a      (      self						,   _A    ):






									'''simple docstring'''







									return self.encoder.get(_A						,   self.encoder[self.unk_token]    )







			def   _a      (      self						,   _A    ):






									'''simple docstring'''







									return self.decoder.get(_A						,   self.unk_token    )







			def   _a      (      self						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		Any				   =			[]
									UpperCamelCase				:		Any				   =			""""""
									for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																					UpperCamelCase				:		Any				   =			self.sp_model.decode(_A    )
																					out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
																					UpperCamelCase				:		List[Any]				   =			[]
															else:
																					current_sub_tokens.append(_A    )
									UpperCamelCase				:		List[Any]				   =			self.sp_model.decode(_A    )
									out_string += decoded.upper() if self.do_upper_case else decoded
									return out_string.strip()







			def   _a      (      self						,   _A						,   _A=None    ):






									'''simple docstring'''







									if token_ids_a is None:
															return self.prefix_tokens + token_ids_a + [self.eos_token_id]
									# We don't expect to process pairs, but leave the pair logic for API consistency
									return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]







			def   _a      (      self						,   _A						,   _A = None						,   _A = False    ):






									'''simple docstring'''







									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=_A						,   token_ids_a=_A						,   already_has_special_tokens=_A    )

									UpperCamelCase				:		Tuple				   =			[1] * len(self.prefix_tokens    )
									UpperCamelCase				:		List[str]				   =			[1]
									if token_ids_a is None:
															return prefix_ones + ([0] * len(_A    )) + suffix_ones
									return prefix_ones + ([0] * len(_A    )) + ([0] * len(_A    )) + suffix_ones







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		List[Any]				   =			self.encoder.copy()
									vocab.update(self.added_tokens_encoder    )
									return vocab







			def __getstate__(      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[int]				   =			self.__dict__.copy()
									UpperCamelCase				:		Optional[Any]				   =			None
									return state







			def __setstate__(      self						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		List[str]				   =			d

									# for backward compatibility
									if not hasattr(self						,   """sp_model_kwargs"""    ):
															UpperCamelCase				:		Optional[int]				   =			{}

									UpperCamelCase				:		int				   =			load_spm(self.spm_file						,   self.sp_model_kwargs    )







			def   _a      (      self						,   _A						,   _A = None    ):






									'''simple docstring'''







									UpperCamelCase				:		int				   =			Path(_A    )
									assert save_dir.is_dir(), f"""{save_directory} should be a directory"""
									UpperCamelCase				:		Dict				   =			save_dir / (
									    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
									)
									UpperCamelCase				:		Dict				   =			save_dir / (
									    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
									)

									save_json(self.encoder						,   _A    )

									if os.path.abspath(self.spm_file    ) != os.path.abspath(_A    ) and os.path.isfile(self.spm_file    ):
															copyfile(self.spm_file						,   _A    )
									elif not os.path.isfile(self.spm_file    ):
															with open(_A						,   """wb"""    ) as fi:
																					UpperCamelCase				:		Tuple				   =			self.sp_model.serialized_model_proto()
																					fi.write(_A    )

									return (str(_A    ), str(_A    ))


def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						UpperCamelCase				:		Dict				   =			sentencepiece.SentencePieceProcessor(**SCREAMING_SNAKE_CASE     )
						spm.Load(str(SCREAMING_SNAKE_CASE     )     )
						return spm

def      UpperCamelCase							(SCREAMING_SNAKE_CASE     ):
						with open(SCREAMING_SNAKE_CASE							,     """r"""     ) as f:
												return json.load(SCREAMING_SNAKE_CASE     )

def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						with open(SCREAMING_SNAKE_CASE							,     """w"""     ) as f:
												json.dump(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     indent=2     )










"""simple docstring"""


from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


__magic_name__       :					List[str]						=			logging.get_logger(__name__)

__magic_name__       :					Any						=			{
    """EleutherAI/gpt-j-6B""": """https://huggingface.co/EleutherAI/gpt-j-6B/resolve/main/config.json""",
    # See all GPT-J models at https://huggingface.co/models?filter=gpt_j
}






class 							lowercase__			(   __SCREAMING_SNAKE_CASE	):

			"""simple docstring"""
			__lowerCAmelCase     :				Optional[int]										=  """gptj"""
			__lowerCAmelCase     :				Optional[Any]										=  {
			    """max_position_embeddings""": """n_positions""",
			    """hidden_size""": """n_embd""",
			    """num_attention_heads""": """n_head""",
			    """num_hidden_layers""": """n_layer""",
			}







			def __init__(      self						,   _A=5_0_4_0_0						,   _A=2_0_4_8						,   _A=4_0_9_6						,   _A=2_8						,   _A=1_6						,   _A=6_4						,   _A=None						,   _A="gelu_new"						,   _A=0.0						,   _A=0.0						,   _A=0.0						,   _A=1e-5						,   _A=0.02						,   _A=True						,   _A=5_0_2_5_6						,   _A=5_0_2_5_6						,   _A=False						,   **_A						,   ):






									'''simple docstring'''







									UpperCamelCase				:		Tuple				   =			vocab_size
									UpperCamelCase				:		Any				   =			n_positions
									UpperCamelCase				:		List[str]				   =			n_embd
									UpperCamelCase				:		List[str]				   =			n_layer
									UpperCamelCase				:		Optional[int]				   =			n_head
									UpperCamelCase				:		int				   =			n_inner
									UpperCamelCase				:		Optional[Any]				   =			rotary_dim
									UpperCamelCase				:		Optional[int]				   =			activation_function
									UpperCamelCase				:		str				   =			resid_pdrop
									UpperCamelCase				:		Union[str, Any]				   =			embd_pdrop
									UpperCamelCase				:		Optional[Any]				   =			attn_pdrop
									UpperCamelCase				:		Optional[int]				   =			layer_norm_epsilon
									UpperCamelCase				:		Any				   =			initializer_range
									UpperCamelCase				:		Optional[int]				   =			use_cache

									UpperCamelCase				:		List[Any]				   =			bos_token_id
									UpperCamelCase				:		List[str]				   =			eos_token_id

									super().__init__(
									    bos_token_id=_A						,   eos_token_id=_A						,   tie_word_embeddings=_A						,   **_A    )






class 							lowercase__			(   __SCREAMING_SNAKE_CASE	):

			"""simple docstring"""







			def __init__(      self						,   _A						,   _A = "default"						,   _A = None						,   _A = False						,   ):






									'''simple docstring'''







									super().__init__(_A						,   task=_A						,   patching_specs=_A						,   use_past=_A    )
									if not getattr(self._config						,   """pad_token_id"""						,   _A    ):
															# TODO: how to do that better?
															UpperCamelCase				:		Optional[Any]				   =			0







			@property
			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		List[str]				   =			OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}    )
									if self.use_past:
															self.fill_with_past_key_values_(_A						,   direction="""inputs"""    )
															UpperCamelCase				:		Any				   =			{0: """batch""", 1: """past_sequence + sequence"""}
									else:
															UpperCamelCase				:		List[Any]				   =			{0: """batch""", 1: """sequence"""}

									return common_inputs







			@property
			def   _a      (      self    ):






									'''simple docstring'''







									return self._config.n_layer







			@property
			def   _a      (      self    ):






									'''simple docstring'''







									return self._config.n_head







			def   _a      (      self						,   _A						,   _A = -1						,   _A = -1						,   _A = False						,   _A = None						,   ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[Any]				   =			super(_A						,   self    ).generate_dummy_inputs(
									    _A						,   batch_size=_A						,   seq_length=_A						,   is_pair=_A						,   framework=_A    )

									# We need to order the input in the way they appears in the forward()
									UpperCamelCase				:		Optional[Any]				   =			OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}    )

									# Need to add the past_keys
									if self.use_past:
															if not is_torch_available():
																					raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""    )
															else:
																					import torch

																					UpperCamelCase   ,						UpperCamelCase				:		Dict				   =			common_inputs["""input_ids"""].shape
																					# Not using the same length for past_key_values
																					UpperCamelCase				:		Dict				   =			seqlen + 2
																					UpperCamelCase				:		Union[str, Any]				   =			(
																					    batch,
																					    self.num_attention_heads,
																					    past_key_values_length,
																					    self._config.hidden_size // self.num_attention_heads,
																					)
																					UpperCamelCase				:		List[Any]				   =			[
																					    (torch.zeros(_A    ), torch.zeros(_A    )) for _ in range(self.num_layers    )
																					]

									UpperCamelCase				:		str				   =			common_inputs["""attention_mask"""]
									if self.use_past:
															UpperCamelCase				:		Any				   =			ordered_inputs["""attention_mask"""].dtype
															UpperCamelCase				:		List[Any]				   =			torch.cat(
															    [ordered_inputs["""attention_mask"""], torch.ones(_A						,   _A						,   dtype=_A    )]						,   dim=1    )

									return ordered_inputs







			@property
			def   _a      (      self    ):






									'''simple docstring'''







									return 1_3



