



"""simple docstring"""

import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class SCREAMING_SNAKE_CASE__       (		__snake_case  ,       unittest.TestCase   ):
						_lowerCAmelCase    						=       LEDTokenizer
						_lowerCAmelCase    						=       LEDTokenizerFast
						_lowerCAmelCase    						=       True



						def      lowerCAmelCase__(self				):





													'''simple docstring'''

													super().setUp()
													__a		:		Any						=							[
													    "l",
													    "o",
													    "w",
													    "e",
													    "r",
													    "s",
													    "t",
													    "i",
													    "d",
													    "n",
													    "\u0120",
													    "\u0120l",
													    "\u0120n",
													    "\u0120lo",
													    "\u0120low",
													    "er",
													    "\u0120lowest",
													    "\u0120newer",
													    "\u0120wider",
													    "<unk>",
													]
													__a		:		int						=							dict(zip(__UpperCAmelCase    ,  range(len(__UpperCAmelCase				)				)				)				)
													__a		:		Dict						=							["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
													__a		:		Tuple						=							{"unk_token": "<unk>"}

													__a		:		Any						=							os.path.join(self.tmpdirname    ,  VOCAB_FILES_NAMES["""vocab_file"""]				)
													__a		:		Union[str, Any]						=							os.path.join(self.tmpdirname    ,  VOCAB_FILES_NAMES["""merges_file"""]				)
													with open(self.vocab_file    ,  """w"""    ,  encoding="""utf-8"""				) as fp:
																				fp.write(json.dumps(__UpperCAmelCase				) + """\n"""				)
													with open(self.merges_file    ,  """w"""    ,  encoding="""utf-8"""				) as fp:
																				fp.write("""\n""".join(__UpperCAmelCase				)				)



						def      lowerCAmelCase__(self    ,  **_lowercase				):





													'''simple docstring'''

													kwargs.update(self.special_tokens_map				)
													return self.tokenizer_class.from_pretrained(self.tmpdirname    ,  **__UpperCAmelCase				)



						def      lowerCAmelCase__(self    ,  **_lowercase				):





													'''simple docstring'''

													kwargs.update(self.special_tokens_map				)
													return self.rust_tokenizer_class.from_pretrained(self.tmpdirname    ,  **__UpperCAmelCase				)



						def      lowerCAmelCase__(self    ,  _lowercase				):





													'''simple docstring'''

													return "lower newer", "lower newer"



						@cached_property
						def      lowerCAmelCase__(self				):





													'''simple docstring'''

													return LEDTokenizer.from_pretrained("""allenai/led-base-16384"""				)



						@cached_property
						def      lowerCAmelCase__(self				):





													'''simple docstring'''

													return LEDTokenizerFast.from_pretrained("""allenai/led-base-16384"""				)



						@require_torch
						def      lowerCAmelCase__(self				):





													'''simple docstring'''

													__a		:		int						=							["A long paragraph for summarization.", "Another paragraph for summarization."]
													__a		:		str						=							[0, 250, 251, 17818, 13, 39186, 1938, 4, 2]

													for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																				__a		:		Union[str, Any]						=							tokenizer(__UpperCAmelCase    ,  max_length=len(__UpperCAmelCase				)    ,  padding=__UpperCAmelCase    ,  return_tensors="""pt"""				)
																				self.assertIsInstance(__UpperCAmelCase    ,  __UpperCAmelCase				)

																				self.assertEqual((2, 9)    ,  batch.input_ids.shape				)
																				self.assertEqual((2, 9)    ,  batch.attention_mask.shape				)
																				__a		:		Any						=							batch.input_ids.tolist()[0]
																				self.assertListEqual(__UpperCAmelCase    ,  __UpperCAmelCase				)



						@require_torch
						def      lowerCAmelCase__(self				):





													'''simple docstring'''

													__a		:		Union[str, Any]						=							["A long paragraph for summarization.", "Another paragraph for summarization."]
													for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																				__a		:		List[str]						=							tokenizer(__UpperCAmelCase    ,  padding=__UpperCAmelCase    ,  return_tensors="""pt"""				)
																				self.assertIn("""input_ids"""    ,  __UpperCAmelCase				)
																				self.assertIn("""attention_mask"""    ,  __UpperCAmelCase				)
																				self.assertNotIn("""labels"""    ,  __UpperCAmelCase				)
																				self.assertNotIn("""decoder_attention_mask"""    ,  __UpperCAmelCase				)



						@require_torch
						def      lowerCAmelCase__(self				):





													'''simple docstring'''

													__a		:		List[Any]						=							[
													    "Summary of the text.",
													    "Another summary.",
													]
													for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																				__a		:		Dict						=							tokenizer(text_target=__UpperCAmelCase    ,  max_length=32    ,  padding="""max_length"""    ,  return_tensors="""pt"""				)
																				self.assertEqual(32    ,  targets["""input_ids"""].shape[1]				)



						@require_torch
						def      lowerCAmelCase__(self				):





													'''simple docstring'''

													for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																				__a		:		int						=							tokenizer(
																				    ["""I am a small frog""" * 1024, """I am a small frog"""]    ,  padding=__UpperCAmelCase    ,  truncation=__UpperCAmelCase    ,  return_tensors="""pt"""				)
																				self.assertIsInstance(__UpperCAmelCase    ,  __UpperCAmelCase				)
																				self.assertEqual(batch.input_ids.shape    ,  (2, 5122)				)



						@require_torch
						def      lowerCAmelCase__(self				):





													'''simple docstring'''

													__a		:		List[str]						=							["A long paragraph for summarization."]
													__a		:		Union[str, Any]						=							[
													    "Summary of the text.",
													]
													for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																				__a		:		str						=							tokenizer(__UpperCAmelCase    ,  return_tensors="""pt"""				)
																				__a		:		List[str]						=							tokenizer(text_target=__UpperCAmelCase    ,  return_tensors="""pt"""				)
																				__a		:		Union[str, Any]						=							inputs["input_ids"]
																				__a		:		Tuple						=							targets["input_ids"]
																				self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()				)
																				self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()				)
																				self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()				)
																				self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()				)



						@require_torch
						def      lowerCAmelCase__(self				):





													'''simple docstring'''

													for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																				__a		:		Tuple						=							["Summary of the text.", "Another summary."]
																				__a		:		Tuple						=							[[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

																				__a		:		str						=							tokenizer(__UpperCAmelCase    ,  padding=__UpperCAmelCase				)
																				__a		:		Optional[int]						=							[[0] * len(__UpperCAmelCase				) for x in encoded_output["input_ids"]]
																				__a		:		Any						=							tokenizer.pad(__UpperCAmelCase				)
																				self.assertSequenceEqual(outputs["""global_attention_mask"""]    ,  __UpperCAmelCase				)



						def      lowerCAmelCase__(self				):





													'''simple docstring'''

													pass





						def      lowerCAmelCase__(self				):





													'''simple docstring'''

													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																				with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
																											__a		:		Any						=							self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase    ,  **__UpperCAmelCase				)
																											__a		:		int						=							self.tokenizer_class.from_pretrained(__UpperCAmelCase    ,  **__UpperCAmelCase				)
																											__a		:		List[str]						=							"A, <mask> AllenNLP sentence."
																											__a		:		Any						=							tokenizer_r.encode_plus(__UpperCAmelCase    ,  add_special_tokens=__UpperCAmelCase    ,  return_token_type_ids=__UpperCAmelCase				)
																											__a		:		Optional[int]						=							tokenizer_p.encode_plus(__UpperCAmelCase    ,  add_special_tokens=__UpperCAmelCase    ,  return_token_type_ids=__UpperCAmelCase				)
																											self.assertEqual(sum(tokens_r["""token_type_ids"""]				)    ,  sum(tokens_p["""token_type_ids"""]				)				)
																											self.assertEqual(
																											    sum(tokens_r["""attention_mask"""]				) / len(tokens_r["""attention_mask"""]				)    ,  sum(tokens_p["""attention_mask"""]				) / len(tokens_p["""attention_mask"""]				)    ,  )

																											__a		:		int						=							tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]				)
																											__a		:		Any						=							tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]				)
																											self.assertSequenceEqual(tokens_p["""input_ids"""]    ,  [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]				)
																											self.assertSequenceEqual(tokens_r["""input_ids"""]    ,  [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]				)

																											self.assertSequenceEqual(
																											    __UpperCAmelCase    ,  ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]				)
																											self.assertSequenceEqual(
																											    __UpperCAmelCase    ,  ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]				)






from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowerCamelCase		       =      logging.get_logger(__name__)


lowerCamelCase		       =      {
    """Salesforce/codegen-350M-nl""": """https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json""",
    """Salesforce/codegen-350M-multi""": """https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json""",
    """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json""",
    """Salesforce/codegen-2B-nl""": """https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json""",
    """Salesforce/codegen-2B-multi""": """https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json""",
    """Salesforce/codegen-2B-mono""": """https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json""",
    """Salesforce/codegen-6B-nl""": """https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json""",
    """Salesforce/codegen-6B-multi""": """https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json""",
    """Salesforce/codegen-6B-mono""": """https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json""",
    """Salesforce/codegen-16B-nl""": """https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json""",
    """Salesforce/codegen-16B-multi""": """https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json""",
    """Salesforce/codegen-16B-mono""": """https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json""",
}


class 				_a						(       SCREAMING_SNAKE_CASE			):





							'''simple docstring'''

							A				:str =       "codegen"
							A				:Tuple =       {
							    "max_position_embeddings": "n_positions",
							    "hidden_size": "n_embd",
							    "num_attention_heads": "n_head",
							    "num_hidden_layers": "n_layer",
							}



							def __init__(							self ,					__UpperCAmelCase=5_0400 ,					__UpperCAmelCase=2048 ,					__UpperCAmelCase=2048 ,					__UpperCAmelCase=4096 ,					__UpperCAmelCase=28 ,					__UpperCAmelCase=16 ,					__UpperCAmelCase=64 ,					__UpperCAmelCase=None ,					__UpperCAmelCase="gelu_new" ,					__UpperCAmelCase=0.0 ,					__UpperCAmelCase=0.0 ,					__UpperCAmelCase=0.0 ,					__UpperCAmelCase=1E-5 ,					__UpperCAmelCase=0.0_2 ,					__UpperCAmelCase=True ,					__UpperCAmelCase=5_0256 ,					__UpperCAmelCase=5_0256 ,					__UpperCAmelCase=False ,					**__UpperCAmelCase ,					):







											"""simple docstring"""



											a__	:							Dict      						= vocab_size
											a__	:							int      						= n_ctx
											a__	:							str      						= n_positions
											a__	:							Union[str, Any]      						= n_embd
											a__	:							Union[str, Any]      						= n_layer
											a__	:							Optional[int]      						= n_head
											a__	:							Dict      						= n_inner
											a__	:							Optional[Any]      						= rotary_dim
											a__	:							str      						= activation_function
											a__	:							List[Any]      						= resid_pdrop
											a__	:							str      						= embd_pdrop
											a__	:							List[str]      						= attn_pdrop
											a__	:							str      						= layer_norm_epsilon
											a__	:							Any      						= initializer_range
											a__	:							List[str]      						= use_cache

											a__	:							Tuple      						= bos_token_id
											a__	:							List[Any]      						= eos_token_id

											super().__init__(
											    bos_token_id=__UpperCAmelCase ,					eos_token_id=__UpperCAmelCase ,					tie_word_embeddings=__UpperCAmelCase ,					**__UpperCAmelCase				)





class 				_a						(       SCREAMING_SNAKE_CASE			):





							'''simple docstring'''

							def __init__(							self ,					__UpperCAmelCase ,					__UpperCAmelCase = "default" ,					__UpperCAmelCase = None ,					__UpperCAmelCase = False ,					):







											"""simple docstring"""



											super().__init__(__UpperCAmelCase ,					task=__UpperCAmelCase ,					patching_specs=__UpperCAmelCase ,					use_past=__UpperCAmelCase				)
											if not getattr(self._config ,					"pad_token_id" ,					__UpperCAmelCase				):
															# TODO: how to do that better?
															a__	:							Union[str, Any]      						= 0



							@property
							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Optional[Any]      						= OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}				)
											if self.use_past:
															self.fill_with_past_key_values_(__UpperCAmelCase ,					direction="inputs"				)
															a__	:							Any      						= {0: "batch", 1: "past_sequence + sequence"}
											else:
															a__	:							List[Any]      						= {0: "batch", 1: "sequence"}

											return common_inputs



							@property
							def 		_A	(							self				):







											"""simple docstring"""



											return self._config.n_layer



							@property
							def 		_A	(							self				):







											"""simple docstring"""



											return self._config.n_head



							def 		_A	(							self ,					__UpperCAmelCase ,					__UpperCAmelCase = -1 ,					__UpperCAmelCase = -1 ,					__UpperCAmelCase = False ,					__UpperCAmelCase = None ,					):







											"""simple docstring"""



											a__	:							Dict      						= super(__UpperCAmelCase ,					self				).generate_dummy_inputs(
											    __UpperCAmelCase ,					batch_size=__UpperCAmelCase ,					seq_length=__UpperCAmelCase ,					is_pair=__UpperCAmelCase ,					framework=__UpperCAmelCase				)

											# We need to order the input in the way they appears in the forward()
											a__	:							Union[str, Any]      						= OrderedDict({"input_ids": common_inputs["input_ids"]}				)

											# Need to add the past_keys
											if self.use_past:
															if not is_torch_available():
																			raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."				)
															else:
																			import torch

																			a__						,   a__	:							Union[str, Any]      						= common_inputs["input_ids"].shape
																			# Not using the same length for past_key_values
																			a__	:							Any      						= seqlen + 2
																			a__	:							str      						= (
																			    batch,
																			    self.num_attention_heads,
																			    past_key_values_length,
																			    self._config.hidden_size // self.num_attention_heads,
																			)
																			a__	:							str      						= [
																			    (torch.zeros(__UpperCAmelCase				), torch.zeros(__UpperCAmelCase				)) for _ in range(self.num_layers				)
																			]

											a__	:							Union[str, Any]      						= common_inputs["attention_mask"]
											if self.use_past:
															a__	:							str      						= ordered_inputs["attention_mask"].dtype
															a__	:							Union[str, Any]      						= torch.cat(
															    [ordered_inputs["attention_mask"], torch.ones(__UpperCAmelCase ,					__UpperCAmelCase ,					dtype=__UpperCAmelCase				)] ,					dim=1				)

											return ordered_inputs




							@property
							def 		_A	(							self				):







											"""simple docstring"""



											return 13


