



from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin



class        lowerCAmelCase__					(  _lowerCamelCase ,				_lowerCamelCase ,				_lowerCamelCase       ):
				A_   :     Union[str, Any]								=     [r'h\.\d+\.attn\.bias', r'h\.\d+\.attn\.masked_bias']


				@register_to_config
				def __init__(							self      :  Optional[Any]						, __UpperCamelCase      :  int						, __UpperCamelCase      :  int						, __UpperCamelCase      :  Optional[int] = None						, __UpperCamelCase      :  int = 50_257						, __UpperCamelCase      :  int = 1_024						, __UpperCamelCase      :  int = 768						, __UpperCamelCase      :  int = 12						, __UpperCamelCase      :  int = 12						, __UpperCamelCase      :  Optional[int] = None						, __UpperCamelCase      :  str = "gelu_new"						, __UpperCamelCase      :  float = 0.1						, __UpperCamelCase      :  float = 0.1						, __UpperCamelCase      :  float = 0.1						, __UpperCamelCase      :  float = 1e-5						, __UpperCamelCase      :  float = 0.0_2						, __UpperCamelCase      :  bool = True						, __UpperCamelCase      :  bool = True						, __UpperCamelCase      :  bool = False						, __UpperCamelCase      :  bool = False						, )		->     int:
											super().__init__()

											A      						=    prefix_length

											if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
																		raise ValueError(
																		    f'''`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'''
																		    f''' `n_embd`: {n_embd} are not equal.'''       )

											A      						=    prefix_inner_dim
											A      						=    prefix_hidden_dim

											A      						=    (
											    nn.Linear(self.prefix_inner_dim						, self.prefix_hidden_dim       )
											    if self.prefix_hidden_dim is not None
											    else nn.Identity()
											)
											A      						=    (
											    nn.Linear(self.prefix_hidden_dim						, __UpperCamelCase       ) if self.prefix_hidden_dim is not None else nn.Identity()
											)

											A      						=    GPTaConfig(
											    vocab_size=__UpperCamelCase						, n_positions=__UpperCamelCase						, n_embd=__UpperCamelCase						, n_layer=__UpperCamelCase						, n_head=__UpperCamelCase						, n_inner=__UpperCamelCase						, activation_function=__UpperCamelCase						, resid_pdrop=__UpperCamelCase						, embd_pdrop=__UpperCamelCase						, attn_pdrop=__UpperCamelCase						, layer_norm_epsilon=__UpperCamelCase						, initializer_range=__UpperCamelCase						, scale_attn_weights=__UpperCamelCase						, use_cache=__UpperCamelCase						, scale_attn_by_inverse_layer_idx=__UpperCamelCase						, reorder_and_upcast_attn=__UpperCamelCase						, )
											A      						=    GPTaLMHeadModel(__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  int						, __UpperCamelCase      :  torch.Tensor						, __UpperCamelCase      :  torch.Tensor						, __UpperCamelCase      :  Optional[torch.Tensor] = None						, __UpperCamelCase      :  Optional[torch.Tensor] = None						, )		->     Optional[Any]:
											A      						=    self.transformer.transformer.wte(__UpperCamelCase       )
											A      						=    self.encode_prefix(__UpperCamelCase       )
											A      						=    self.decode_prefix(__UpperCamelCase       )
											A      						=    torch.cat((prefix_embeds, embedding_text)						, dim=1       )

											if labels is not None:
																		A      						=    self.get_dummy_token(input_ids.shape[0]						, input_ids.device       )
																		A      						=    torch.cat((dummy_token, input_ids)						, dim=1       )
											A      						=    self.transformer(inputs_embeds=__UpperCamelCase						, labels=__UpperCamelCase						, attention_mask=__UpperCamelCase       )
											if self.prefix_hidden_dim is not None:
																		return out, hidden
											else:
																		return out


				def __UpperCamelCase						(							self      :  Dict						, __UpperCamelCase      :  int						, __UpperCamelCase      :  torch.device       )		->     torch.Tensor:
											return torch.zeros(__UpperCamelCase						, self.prefix_length						, dtype=torch.intaa						, device=__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Optional[int]						, __UpperCamelCase      :  Tuple       )		->     Optional[int]:
											return self.encode_prefix(__UpperCamelCase       )


				@torch.no_grad()
				def __UpperCamelCase						(							self      :  Union[str, Any]						, __UpperCamelCase      :  str						, __UpperCamelCase      :  Union[str, Any]						, __UpperCamelCase      :  Optional[Any]       )		->     Dict:
											A      						=    torch.split(__UpperCamelCase						, 1						, dim=0       )
											A      						=    []
											A      						=    []
											for feature in features:
																		A      						=    self.decode_prefix(feature.to(__UpperCamelCase       )       )  # back to the clip feature
																		# Only support beam search for now
																		A						,   A      						=    self.generate_beam(
																		    input_embeds=__UpperCamelCase						, device=__UpperCamelCase						, eos_token_id=__UpperCamelCase       )
																		generated_tokens.append(output_tokens[0]       )
																		generated_seq_lengths.append(seq_lengths[0]       )
											A      						=    torch.stack(__UpperCamelCase       )
											A      						=    torch.stack(__UpperCamelCase       )
											return generated_tokens, generated_seq_lengths


				@torch.no_grad()
				def __UpperCamelCase						(							self      :  Optional[int]						, __UpperCamelCase      :  Optional[int]=None						, __UpperCamelCase      :  Optional[Any]=None						, __UpperCamelCase      :  int=None						, __UpperCamelCase      :  int = 5						, __UpperCamelCase      :  int = 67						, __UpperCamelCase      :  float = 1.0						, __UpperCamelCase      :  Optional[int] = None						, )		->     Tuple:
											A      						=    eos_token_id
											A      						=    None
											A      						=    None
											A      						=    torch.ones(__UpperCamelCase						, device=__UpperCamelCase						, dtype=torch.int       )
											A      						=    torch.zeros(__UpperCamelCase						, device=__UpperCamelCase						, dtype=torch.bool       )

											if input_embeds is not None:
																		A      						=    input_embeds
											else:
																		A      						=    self.transformer.transformer.wte(__UpperCamelCase       )

											for i in range(__UpperCamelCase       ):
																		A      						=    self.transformer(inputs_embeds=__UpperCamelCase       )
																		A      						=    outputs.logits
																		A      						=    logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
																		A      						=    logits.softmax(-1       ).log()

																		if scores is None:
																									A						,   A      						=    logits.topk(__UpperCamelCase						, -1       )
																									A      						=    generated.expand(__UpperCamelCase						, *generated.shape[1:]       )
																									A						,   A      						=    next_tokens.permute(1						, 0       ), scores.squeeze(0       )
																									if tokens is None:
																																A      						=    next_tokens
																									else:
																																A      						=    tokens.expand(__UpperCamelCase						, *tokens.shape[1:]       )
																																A      						=    torch.cat((tokens, next_tokens)						, dim=1       )
																		else:
																									A      						=    -float(np.inf       )
																									A      						=    0
																									A      						=    scores[:, None] + logits
																									seq_lengths[~is_stopped] += 1
																									A      						=    scores_sum / seq_lengths[:, None]
																									A						,   A      						=    scores_sum_average.view(-1       ).topk(__UpperCamelCase						, -1       )
																									A      						=    next_tokens // scores_sum.shape[1]
																									A      						=    seq_lengths[next_tokens_source]
																									A      						=    next_tokens % scores_sum.shape[1]
																									A      						=    next_tokens.unsqueeze(1       )
																									A      						=    tokens[next_tokens_source]
																									A      						=    torch.cat((tokens, next_tokens)						, dim=1       )
																									A      						=    generated[next_tokens_source]
																									A      						=    scores_sum_average * seq_lengths
																									A      						=    is_stopped[next_tokens_source]

																		A      						=    self.transformer.transformer.wte(next_tokens.squeeze()       ).view(generated.shape[0]						, 1						, -1       )
																		A      						=    torch.cat((generated, next_token_embed)						, dim=1       )
																		A      						=    is_stopped + next_tokens.eq(__UpperCamelCase       ).squeeze()
																		if is_stopped.all():
																									break

											A      						=    scores / seq_lengths
											A      						=    scores.argsort(descending=__UpperCamelCase       )
											# tokens tensors are already padded to max_seq_length
											A      						=    [tokens[i] for i in order]
											A      						=    torch.stack(__UpperCamelCase						, dim=0       )
											A      						=    torch.tensor([seq_lengths[i] for i in order]						, dtype=seq_lengths.dtype       )
											return output_texts, seq_lengths




import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class        lowerCAmelCase__					(  _lowerCamelCase ,				unittest.TestCase       ):
				A_   :     Dict								=     GPTSanJapaneseTokenizer
				A_   :     Optional[int]								=     False
				A_   :     List[str]								=     {'do_clean_text': False, 'add_prefix_space': False}


				def __UpperCamelCase						(							self      :  Tuple       )		->     Any:
											super().setUp()

											# fmt: off
											A      						=    ['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
											# fmt: on
											A      						=    {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
											A      						=    {'unk_token': '<unk>'}

											A      						=    os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES['vocab_file']       )
											A      						=    os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES['emoji_file']       )
											with open(self.vocab_file						, 'w'						, encoding='utf-8'       ) as vocab_writer:
																		vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )
											with open(self.emoji_file						, 'w'       ) as emoji_writer:
																		emoji_writer.write(json.dumps(__UpperCamelCase       )       )


				def __UpperCamelCase						(							self      :  Any						, **__UpperCamelCase      :  Any       )		->     Tuple:
											kwargs.update(self.special_tokens_map       )
											return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname						, **__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  Dict       )		->     List[str]:
											A      						=    'こんにちは、世界。 \nこんばんは、㔺界。😀'
											A      						=    'こんにちは、世界。 \nこんばんは、世界。😀'
											return input_text, output_text


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  Optional[Any]       )		->     List[str]:
											A						,   A      						=    self.get_input_output_texts(__UpperCamelCase       )
											A      						=    tokenizer.encode(__UpperCamelCase						, add_special_tokens=__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase						, clean_up_tokenization_spaces=__UpperCamelCase       )
											return text, ids


				def __UpperCamelCase						(							self      :  Any       )		->     int:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     Tuple:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     List[Any]:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     int:
											A      						=    self.get_tokenizer()

											# Testing tokenization
											A      						=    'こんにちは、世界。　こんばんは、㔺界。'
											A      						=    ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
											A      						=    tokenizer.tokenize(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )

											# Testing conversion to ids without special tokens
											A      						=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
											A      						=    tokenizer.convert_tokens_to_ids(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )

											# Testing conversion to ids with special tokens
											A      						=    tokens + [tokenizer.unk_token]
											A      						=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
											A      						=    tokenizer.convert_tokens_to_ids(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[str]:
											A      						=    self.get_tokenizer()

											# Testing tokenization
											A      						=    'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
											A      						=    'こんにちは、、、、世界。こんばんは、、、、世界。'
											A      						=    tokenizer.encode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     str:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											# Testing tokenization
											A      						=    'こんにちは、世界。'
											A      						=    'こんばんは、㔺界。😀'
											A      						=    'こんにちは、世界。こんばんは、世界。😀'
											A      						=    tokenizer.encode(prefix_text + input_text       )
											A      						=    tokenizer.encode(''						, prefix_text=prefix_text + input_text       )
											A      						=    tokenizer.encode(__UpperCamelCase						, prefix_text=__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[Any]:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											# Testing tokenization
											A      						=    'こんにちは、世界。'
											A      						=    'こんばんは、㔺界。😀'

											A      						=    len(tokenizer.encode(__UpperCamelCase       )       ) - 2
											A      						=    len(tokenizer.encode(__UpperCamelCase       )       ) - 2

											A      						=    [1] + [0] * (len_prefix + len_text + 1)
											A      						=    [1] * (len_prefix + len_text + 1) + [0]
											A      						=    [1] + [1] * (len_prefix) + [0] * (len_text + 1)

											A      						=    tokenizer(prefix_text + input_text       ).token_type_ids
											A      						=    tokenizer(''						, prefix_text=prefix_text + input_text       ).token_type_ids
											A      						=    tokenizer(__UpperCamelCase						, prefix_text=__UpperCamelCase       ).token_type_ids
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Any       )		->     str:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											A      						=    tokenizer.encode('あンいワ'       )
											A      						=    tokenizer.encode(''						, prefix_text='あンいワ'       )
											A      						=    tokenizer.encode('いワ'						, prefix_text='あン'       )

											self.assertEqual(tokenizer.decode(__UpperCamelCase       )						, tokenizer.decode(__UpperCamelCase       )       )
											self.assertEqual(tokenizer.decode(__UpperCamelCase       )						, tokenizer.decode(__UpperCamelCase       )       )
											self.assertNotEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertNotEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(x_token_a[1]						, x_token_a[-1]       )  # SEG token
											self.assertEqual(x_token_a[1]						, x_token_a[3]       )  # SEG token


				@slow
				def __UpperCamelCase						(							self      :  Any       )		->     List[Any]:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											A      						=    [['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
											A      						=    tokenizer(__UpperCamelCase						, padding=__UpperCamelCase       )
											A      						=    tokenizer.batch_encode_plus(__UpperCamelCase						, padding=__UpperCamelCase       )

											# fmt: off
											A      						=    [[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
											A      						=    [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
											A      						=    [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
											# fmt: on
											self.assertListEqual(x_token.input_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token.token_type_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token.attention_mask						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.input_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.token_type_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.attention_mask						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Tuple       )		->     str:
											# Intentionally convert some words to accommodate character fluctuations unique to Japanese
											pass


				def __UpperCamelCase						(							self      :  Dict       )		->     int:
											# tokenizer has no padding token
											pass