




"""simple docstring"""




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase				:     Union[str, Any]					=				logging.get_logger(__name__)

lowerCAmelCase				:     List[str]					=				"""▁"""

lowerCAmelCase				:     Any					=				{"""vocab_file""": """sentencepiece.bpe.model"""}

lowerCAmelCase				:     str					=				{
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

lowerCAmelCase				:     Optional[int]					=				{
    """facebook/mbart-large-en-ro""": 1024,
    """facebook/mbart-large-cc25""": 1024,
}

# fmt: off
lowerCAmelCase				:     Dict					=				["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]






class        __magic_name__				(			UpperCAmelCase__							):






				'''simple docstring'''
				__UpperCamelCase         =		VOCAB_FILES_NAMES
				__UpperCamelCase         =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase         =		PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase         =		["input_ids", "attention_mask"]

				__UpperCamelCase         =		[]
				__UpperCamelCase         =		[]





				def __init__(			self						,					_a						,					_a="<s>"						,					_a="</s>"						,					_a="</s>"						,					_a="<s>"						,					_a="<unk>"						,					_a="<pad>"						,					_a="<mask>"						,					_a=None						,					_a=None						,					_a=None						,					_a = None						,					_a=None						,					**_a						,					):





							"""simple docstring"""
							# Mask token behave like a normal word, i.e. include the space before it
							lowerCamelCase      =   AddedToken(_a						,					lstrip=_a						,					rstrip=_a					) if isinstance(_a						,					_a					) else mask_token

							lowerCamelCase      =   {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=_a						,					eos_token=_a						,					unk_token=_a						,					sep_token=_a						,					cls_token=_a						,					pad_token=_a						,					mask_token=_a						,					tokenizer_file=_a						,					src_lang=_a						,					tgt_lang=_a						,					additional_special_tokens=_a						,					sp_model_kwargs=self.sp_model_kwargs						,					**_a						,					)

							lowerCamelCase      =   spm.SentencePieceProcessor(**self.sp_model_kwargs					)
							self.sp_model.Load(str(_a					)					)
							lowerCamelCase      =   vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# Mimic fairseq token-to-id alignment for the first 4 token
							lowerCamelCase      =   {"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							lowerCamelCase      =   1

							lowerCamelCase      =   len(self.sp_model					)
							lowerCamelCase      =   {
							    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_a					)
							}
							lowerCamelCase      =   {v: k for k, v in self.lang_code_to_id.items()}
							lowerCamelCase      =   len(self.sp_model					) + len(self.lang_code_to_id					) + self.fairseq_offset

							self.fairseq_tokens_to_ids.update(self.lang_code_to_id					)
							lowerCamelCase      =   {v: k for k, v in self.fairseq_tokens_to_ids.items()}
							lowerCamelCase      =   list(self.lang_code_to_id.keys()					)

							if additional_special_tokens is not None:
										# Only add those special tokens if they are not already there.
										self._additional_special_tokens.extend(
										    [t for t in additional_special_tokens if t not in self._additional_special_tokens]					)

							lowerCamelCase      =   src_lang if src_lang is not None else """en_XX"""
							lowerCamelCase      =   self.lang_code_to_id[self._src_lang]
							lowerCamelCase      =   tgt_lang
							self.set_src_lang_special_tokens(self._src_lang					)





				def __getstate__(			self					):





							"""simple docstring"""
							lowerCamelCase      =   self.__dict__.copy()
							lowerCamelCase      =   None
							lowerCamelCase      =   self.sp_model.serialized_model_proto()
							return state





				def __setstate__(			self						,					_a					):





							"""simple docstring"""
							lowerCamelCase      =   d

							# for backward compatibility
							if not hasattr(self						,					"""sp_model_kwargs"""					):
										lowerCamelCase      =   {}

							lowerCamelCase      =   spm.SentencePieceProcessor(**self.sp_model_kwargs					)
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto					)





				@property
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							return len(self.sp_model					) + len(self.lang_code_to_id					) + self.fairseq_offset + 1  # Plus 1 for the mask token





				@property
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							return self._src_lang





				@src_lang.setter
				def   _lowerCAmelCase							(			self						,					_a					):





							"""simple docstring"""
							lowerCamelCase      =   new_src_lang
							self.set_src_lang_special_tokens(self._src_lang					)





				def   _lowerCAmelCase							(			self						,					_a						,					_a = None						,					_a = False					):





							"""simple docstring"""

							if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=_a						,					token_ids_a=_a						,					already_has_special_tokens=_a					)

							lowerCamelCase      =   [1] * len(self.prefix_tokens					)
							lowerCamelCase      =   [1] * len(self.suffix_tokens					)
							if token_ids_a is None:
										return prefix_ones + ([0] * len(_a					)) + suffix_ones
							return prefix_ones + ([0] * len(_a					)) + ([0] * len(_a					)) + suffix_ones





				def   _lowerCAmelCase							(			self						,					_a						,					_a = None					):





							"""simple docstring"""
							if token_ids_a is None:
										return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





				def   _lowerCAmelCase							(			self						,					_a						,					_a = None					):





							"""simple docstring"""
							lowerCamelCase      =   [self.sep_token_id]
							lowerCamelCase      =   [self.cls_token_id]

							if token_ids_a is None:
										return len(cls + token_ids_a + sep					) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]





				def   _lowerCAmelCase							(			self						,					_a						,					_a						,					_a						,					_a						,					**_a					):





							"""simple docstring"""
							if src_lang is None or tgt_lang is None:
										raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""					)
							lowerCamelCase      =   src_lang
							lowerCamelCase      =   self(_a						,					add_special_tokens=_a						,					return_tensors=_a						,					**_a					)
							lowerCamelCase      =   self.convert_tokens_to_ids(_a					)
							lowerCamelCase      =   tgt_lang_id
							return inputs





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   {self.convert_ids_to_tokens(_a					): i for i in range(self.vocab_size					)}
							vocab.update(self.added_tokens_encoder					)
							return vocab





				def   _lowerCAmelCase							(			self						,					_a					):





							"""simple docstring"""
							return self.sp_model.encode(_a						,					out_type=_a					)





				def   _lowerCAmelCase							(			self						,					_a					):





							"""simple docstring"""
							if token in self.fairseq_tokens_to_ids:
										return self.fairseq_tokens_to_ids[token]
							lowerCamelCase      =   self.sp_model.PieceToId(_a					)

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id





				def   _lowerCAmelCase							(			self						,					_a					):





							"""simple docstring"""
							if index in self.fairseq_ids_to_tokens:
										return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset					)





				def   _lowerCAmelCase							(			self						,					_a					):





							"""simple docstring"""
							lowerCamelCase      =   """""".join(_a					).replace(_a						,					""" """					).strip()
							return out_string





				def   _lowerCAmelCase							(			self						,					_a						,					_a = None					):





							"""simple docstring"""
							if not os.path.isdir(_a					):
										logger.error(f'Vocabulary path ({save_directory}) should be a directory'					)
										return
							lowerCamelCase      =   os.path.join(
							    _a						,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)

							if os.path.abspath(self.vocab_file					) != os.path.abspath(_a					) and os.path.isfile(self.vocab_file					):
										copyfile(self.vocab_file						,					_a					)
							elif not os.path.isfile(self.vocab_file					):
										with open(_a						,					"""wb"""					) as fi:
													lowerCamelCase      =   self.sp_model.serialized_model_proto()
													fi.write(_a					)

							return (out_vocab_file,)





				def   _lowerCAmelCase							(			self						,					_a						,					_a = "en_XX"						,					_a = None						,					_a = "ro_RO"						,					**_a						,					):





							"""simple docstring"""
							lowerCamelCase      =   src_lang
							lowerCamelCase      =   tgt_lang
							return super().prepare_seqaseq_batch(_a						,					_a						,					**_a					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							return self.set_src_lang_special_tokens(self.src_lang					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							return self.set_tgt_lang_special_tokens(self.tgt_lang					)





				def   _lowerCAmelCase							(			self						,					_a					):





							"""simple docstring"""
							lowerCamelCase      =   self.lang_code_to_id[src_lang]
							lowerCamelCase      =   []
							lowerCamelCase      =   [self.eos_token_id, self.cur_lang_code]





				def   _lowerCAmelCase							(			self						,					_a					):





							"""simple docstring"""
							lowerCamelCase      =   self.lang_code_to_id[lang]
							lowerCamelCase      =   []
							lowerCamelCase      =   [self.eos_token_id, self.cur_lang_code]









"""simple docstring"""




from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin






class        __magic_name__				(			UpperCAmelCase__		,   UpperCAmelCase__		,   UpperCAmelCase__							):






				'''simple docstring'''
				__UpperCamelCase         =		[R"h\.\d+\.attn\.bias", R"h\.\d+\.attn\.masked_bias"]





				@register_to_config
				def __init__(			self						,					_a						,					_a						,					_a = None						,					_a = 50_257						,					_a = 1_024						,					_a = 768						,					_a = 12						,					_a = 12						,					_a = None						,					_a = "gelu_new"						,					_a = 0.1						,					_a = 0.1						,					_a = 0.1						,					_a = 1e-5						,					_a = 0.02						,					_a = True						,					_a = True						,					_a = False						,					_a = False						,					):





							"""simple docstring"""
							super().__init__()

							lowerCamelCase      =   prefix_length

							if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
										raise ValueError(
										    f'`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'
										    f' `n_embd`: {n_embd} are not equal.'					)

							lowerCamelCase      =   prefix_inner_dim
							lowerCamelCase      =   prefix_hidden_dim

							lowerCamelCase      =   (
							    nn.Linear(self.prefix_inner_dim						,					self.prefix_hidden_dim					)
							    if self.prefix_hidden_dim is not None
							    else nn.Identity()
							)
							lowerCamelCase      =   (
							    nn.Linear(self.prefix_hidden_dim						,					_a					) if self.prefix_hidden_dim is not None else nn.Identity()
							)

							lowerCamelCase      =   GPTaConfig(
							    vocab_size=_a						,					n_positions=_a						,					n_embd=_a						,					n_layer=_a						,					n_head=_a						,					n_inner=_a						,					activation_function=_a						,					resid_pdrop=_a						,					embd_pdrop=_a						,					attn_pdrop=_a						,					layer_norm_epsilon=_a						,					initializer_range=_a						,					scale_attn_weights=_a						,					use_cache=_a						,					scale_attn_by_inverse_layer_idx=_a						,					reorder_and_upcast_attn=_a						,					)
							lowerCamelCase      =   GPTaLMHeadModel(_a					)





				def   _lowerCAmelCase							(			self						,					_a						,					_a						,					_a = None						,					_a = None						,					):





							"""simple docstring"""
							lowerCamelCase      =   self.transformer.transformer.wte(_a					)
							lowerCamelCase      =   self.encode_prefix(_a					)
							lowerCamelCase      =   self.decode_prefix(_a					)
							lowerCamelCase      =   torch.cat((prefix_embeds, embedding_text)						,					dim=1					)

							if labels is not None:
										lowerCamelCase      =   self.get_dummy_token(input_ids.shape[0]						,					input_ids.device					)
										lowerCamelCase      =   torch.cat((dummy_token, input_ids)						,					dim=1					)
							lowerCamelCase      =   self.transformer(inputs_embeds=_a						,					labels=_a						,					attention_mask=_a					)
							if self.prefix_hidden_dim is not None:
										return out, hidden
							else:
										return out





				def   _lowerCAmelCase							(			self						,					_a						,					_a					):





							"""simple docstring"""
							return torch.zeros(_a						,					self.prefix_length						,					dtype=torch.intaa						,					device=_a					)





				def   _lowerCAmelCase							(			self						,					_a					):





							"""simple docstring"""
							return self.encode_prefix(_a					)





				@torch.no_grad()
				def   _lowerCAmelCase							(			self						,					_a						,					_a						,					_a					):





							"""simple docstring"""
							lowerCamelCase      =   torch.split(_a						,					1						,					dim=0					)
							lowerCamelCase      =   []
							lowerCamelCase      =   []
							for feature in features:
										lowerCamelCase      =   self.decode_prefix(feature.to(_a					)					)  # back to the clip feature
										# Only support beam search for now
										lowerCamelCase ,  lowerCamelCase      =   self.generate_beam(
										    input_embeds=_a						,					device=_a						,					eos_token_id=_a					)
										generated_tokens.append(output_tokens[0]					)
										generated_seq_lengths.append(seq_lengths[0]					)
							lowerCamelCase      =   torch.stack(_a					)
							lowerCamelCase      =   torch.stack(_a					)
							return generated_tokens, generated_seq_lengths





				@torch.no_grad()
				def   _lowerCAmelCase							(			self						,					_a=None						,					_a=None						,					_a=None						,					_a = 5						,					_a = 67						,					_a = 1.0						,					_a = None						,					):





							"""simple docstring"""
							lowerCamelCase      =   eos_token_id
							lowerCamelCase      =   None
							lowerCamelCase      =   None
							lowerCamelCase      =   torch.ones(_a						,					device=_a						,					dtype=torch.int					)
							lowerCamelCase      =   torch.zeros(_a						,					device=_a						,					dtype=torch.bool					)

							if input_embeds is not None:
										lowerCamelCase      =   input_embeds
							else:
										lowerCamelCase      =   self.transformer.transformer.wte(_a					)

							for i in range(_a					):
										lowerCamelCase      =   self.transformer(inputs_embeds=_a					)
										lowerCamelCase      =   outputs.logits
										lowerCamelCase      =   logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
										lowerCamelCase      =   logits.softmax(-1					).log()

										if scores is None:
													lowerCamelCase ,  lowerCamelCase      =   logits.topk(_a						,					-1					)
													lowerCamelCase      =   generated.expand(_a						,					*generated.shape[1:]					)
													lowerCamelCase ,  lowerCamelCase      =   next_tokens.permute(1						,					0					), scores.squeeze(0					)
													if tokens is None:
																lowerCamelCase      =   next_tokens
													else:
																lowerCamelCase      =   tokens.expand(_a						,					*tokens.shape[1:]					)
																lowerCamelCase      =   torch.cat((tokens, next_tokens)						,					dim=1					)
										else:
													lowerCamelCase      =   -float(np.inf					)
													lowerCamelCase      =   0
													lowerCamelCase      =   scores[:, None] + logits
													seq_lengths[~is_stopped] += 1
													lowerCamelCase      =   scores_sum / seq_lengths[:, None]
													lowerCamelCase ,  lowerCamelCase      =   scores_sum_average.view(-1					).topk(_a						,					-1					)
													lowerCamelCase      =   next_tokens // scores_sum.shape[1]
													lowerCamelCase      =   seq_lengths[next_tokens_source]
													lowerCamelCase      =   next_tokens % scores_sum.shape[1]
													lowerCamelCase      =   next_tokens.unsqueeze(1					)
													lowerCamelCase      =   tokens[next_tokens_source]
													lowerCamelCase      =   torch.cat((tokens, next_tokens)						,					dim=1					)
													lowerCamelCase      =   generated[next_tokens_source]
													lowerCamelCase      =   scores_sum_average * seq_lengths
													lowerCamelCase      =   is_stopped[next_tokens_source]

										lowerCamelCase      =   self.transformer.transformer.wte(next_tokens.squeeze()					).view(generated.shape[0]						,					1						,					-1					)
										lowerCamelCase      =   torch.cat((generated, next_token_embed)						,					dim=1					)
										lowerCamelCase      =   is_stopped + next_tokens.eq(_a					).squeeze()
										if is_stopped.all():
													break

							lowerCamelCase      =   scores / seq_lengths
							lowerCamelCase      =   scores.argsort(descending=_a					)
							# tokens tensors are already padded to max_seq_length
							lowerCamelCase      =   [tokens[i] for i in order]
							lowerCamelCase      =   torch.stack(_a						,					dim=0					)
							lowerCamelCase      =   torch.tensor([seq_lengths[i] for i in order]						,					dtype=seq_lengths.dtype					)
							return output_texts, seq_lengths



