




import tensorflow as tf

from ...tf_utils import shape_list

class 						__lowerCAmelCase      (      tf.keras.layers.Layer					):
							def __init__(		self		:Optional[Any]						, __magic_name__		:Optional[int]						, __magic_name__		:Any						, __magic_name__		:Any						, __magic_name__		:List[str]						, __magic_name__		:Dict=1						, __magic_name__		:Optional[int]=False						, **__magic_name__		:Tuple							):







								'''simple docstring'''







								super().__init__(**__magic_name__							)

								a									=				vocab_size
								a									=				d_embed
								a									=				d_proj

								a									=				cutoffs + [vocab_size]
								a									=				[0] + self.cutoffs
								a									=				div_val

								a									=				self.cutoffs[0]
								a									=				len(self.cutoffs							) - 1
								a									=				self.shortlist_size + self.n_clusters
								a									=				keep_order

								a									=				[]
								a									=				[]
							def  lowerCamelCase__				(		self		:Union[str, Any]						, __magic_name__		:str							):







								'''simple docstring'''







								if self.n_clusters > 0:
									a									=				self.add_weight(
									    shape=(self.n_clusters, self.d_embed)						, initializer="""zeros"""						, trainable=__magic_name__						, name="""cluster_weight"""							)
									a									=				self.add_weight(
									    shape=(self.n_clusters,)						, initializer="""zeros"""						, trainable=__magic_name__						, name="""cluster_bias"""							)

								if self.div_val == 1:
									for i in range(len(self.cutoffs							)							):
										if self.d_proj != self.d_embed:
											a									=				self.add_weight(
											    shape=(self.d_embed, self.d_proj)						, initializer="""zeros"""						, trainable=__magic_name__						, name=F'out_projs_._{i}'						, )
											self.out_projs.append(__magic_name__							)
										else:
											self.out_projs.append(__magic_name__							)
										a									=				self.add_weight(
										    shape=(self.vocab_size, self.d_embed)						, initializer="""zeros"""						, trainable=__magic_name__						, name=F'out_layers_._{i}_._weight'						, )
										a									=				self.add_weight(
										    shape=(self.vocab_size,)						, initializer="""zeros"""						, trainable=__magic_name__						, name=F'out_layers_._{i}_._bias'						, )
										self.out_layers.append((weight, bias)							)
								else:
									for i in range(len(self.cutoffs							)							):
										a							,					a									=				self.cutoff_ends[i], self.cutoff_ends[i + 1]
										a									=				self.d_embed // (self.div_val**i)

										a									=				self.add_weight(
										    shape=(d_emb_i, self.d_proj)						, initializer="""zeros"""						, trainable=__magic_name__						, name=F'out_projs_._{i}'							)
										self.out_projs.append(__magic_name__							)
										a									=				self.add_weight(
										    shape=(r_idx - l_idx, d_emb_i)						, initializer="""zeros"""						, trainable=__magic_name__						, name=F'out_layers_._{i}_._weight'						, )
										a									=				self.add_weight(
										    shape=(r_idx - l_idx,)						, initializer="""zeros"""						, trainable=__magic_name__						, name=F'out_layers_._{i}_._bias'						, )
										self.out_layers.append((weight, bias)							)
								super().build(__magic_name__							)
							@staticmethod
							def  lowerCamelCase__				(		__magic_name__		:List[str]						, __magic_name__		:Optional[int]						, __magic_name__		:Union[str, Any]						, __magic_name__		:int=None							):







								'''simple docstring'''







								a									=				x
								if proj is not None:
									a									=				tf.einsum("""ibd,ed->ibe"""						, __magic_name__						, __magic_name__							)
								return tf.einsum("""ibd,nd->ibn"""						, __magic_name__						, __magic_name__							) + b
							@staticmethod
							def  lowerCamelCase__				(		__magic_name__		:str						, __magic_name__		:Optional[int]							):







								'''simple docstring'''







								a									=				shape_list(__magic_name__							)
								a									=				tf.range(lp_size[0]						, dtype=target.dtype							)
								a									=				tf.stack([r, target]						, 1							)
								return tf.gather_nd(__magic_name__						, __magic_name__							)





							def  lowerCamelCase__				(		self		:Any						, __magic_name__		:int						, __magic_name__		:Optional[int]						, __magic_name__		:List[str]=True						, __magic_name__		:str=False							):







								'''simple docstring'''







								a									=				0
								if self.n_clusters == 0:
									a									=				self._logit(__magic_name__						, self.out_layers[0][0]						, self.out_layers[0][1]						, self.out_projs[0]							)
									if target is not None:
										a									=				tf.nn.sparse_softmax_cross_entropy_with_logits(labels=__magic_name__						, logits=__magic_name__							)
									a									=				tf.nn.log_softmax(__magic_name__						, axis=-1							)
								else:
									a									=				shape_list(__magic_name__							)
									a									=				[]
									a									=				tf.zeros(hidden_sizes[:2]							)
									for i in range(len(self.cutoffs							)							):
										a							,					a									=				self.cutoff_ends[i], self.cutoff_ends[i + 1]
										if target is not None:
											a									=				(target >= l_idx) & (target < r_idx)
											a									=				tf.where(__magic_name__							)
											a									=				tf.boolean_mask(__magic_name__						, __magic_name__							) - l_idx

										if self.div_val == 1:
											a									=				self.out_layers[0][0][l_idx:r_idx]
											a									=				self.out_layers[0][1][l_idx:r_idx]
										else:
											a									=				self.out_layers[i][0]
											a									=				self.out_layers[i][1]

										if i == 0:
											a									=				tf.concat([cur_W, self.cluster_weight]						, 0							)
											a									=				tf.concat([cur_b, self.cluster_bias]						, 0							)

											a									=				self._logit(__magic_name__						, __magic_name__						, __magic_name__						, self.out_projs[0]							)
											a									=				tf.nn.log_softmax(__magic_name__							)
											out.append(head_logprob[..., : self.cutoffs[0]]							)
											if target is not None:
												a									=				tf.boolean_mask(__magic_name__						, __magic_name__							)
												a									=				self._gather_logprob(__magic_name__						, __magic_name__							)
										else:
											a									=				self._logit(__magic_name__						, __magic_name__						, __magic_name__						, self.out_projs[i]							)
											a									=				tf.nn.log_softmax(__magic_name__							)
											a									=				self.cutoffs[0] + i - 1  # No probability for the head cluster
											a									=				head_logprob[..., cluster_prob_idx, None] + tail_logprob
											out.append(__magic_name__							)
											if target is not None:
												a									=				tf.boolean_mask(__magic_name__						, __magic_name__							)
												a									=				tf.boolean_mask(__magic_name__						, __magic_name__							)
												a									=				self._gather_logprob(__magic_name__						, __magic_name__							)
												cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
										if target is not None:
											loss += tf.scatter_nd(__magic_name__						, -cur_logprob						, shape_list(__magic_name__							)							)
									a									=				tf.concat(__magic_name__						, axis=-1							)

								if target is not None:
									if return_mean:
										a									=				tf.reduce_mean(__magic_name__							)
									# Add the training-time loss value to the layer using `self.add_loss()`.
									self.add_loss(__magic_name__							)

									# Log the loss as a metric (we could log arbitrary metrics,
									# including different metrics for training and inference.
									self.add_metric(__magic_name__						, name=self.name						, aggregation="""mean""" if return_mean else """"""							)

								return out







import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin

class 						__lowerCAmelCase      (      __magic_name__ ,			unittest.TestCase					):
							UpperCamelCase__   	=    TransfoXLTokenizer
							UpperCamelCase__   	=    False
							UpperCamelCase__   	=    False
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								super().setUp()

								a									=				[
								    """<unk>""",
								    """[CLS]""",
								    """[SEP]""",
								    """want""",
								    """unwanted""",
								    """wa""",
								    """un""",
								    """running""",
								    """,""",
								    """low""",
								    """l""",
								]
								a									=				os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES["""vocab_file"""]							)
								with open(self.vocab_file						, """w"""						, encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)
							def  lowerCamelCase__				(		self		:List[Any]						, **__magic_name__		:Optional[int]							):







								'''simple docstring'''







								a									=				True
								return TransfoXLTokenizer.from_pretrained(self.tmpdirname						, **__magic_name__							)
							def  lowerCamelCase__				(		self		:Union[str, Any]						, __magic_name__		:int							):







								'''simple docstring'''







								a									=				"""<unk> UNwanted , running"""
								a									=				"""<unk> unwanted, running"""
								return input_text, output_text
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(vocab_file=self.vocab_file						, lower_case=__magic_name__							)

								a									=				tokenizer.tokenize("""<unk> UNwanted , running"""							)
								self.assertListEqual(__magic_name__						, ["""<unk>""", """unwanted""", """,""", """running"""]							)

								self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__							)						, [0, 4, 8, 7]							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(lower_case=__magic_name__							)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """							)						, ["""hello""", """!""", """how""", """are""", """you""", """?"""]							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(lower_case=__magic_name__							)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """							)						, ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)
							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(lower_case=__magic_name__							)
								a									=				"""Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
								a									=				[
								    """Hello""",
								    """(""",
								    """bracket""",
								    """)""",
								    """and""",
								    """side""",
								    """@-@""",
								    """scrolled""",
								    """[""",
								    """and""",
								    """]""",
								    """Henry""",
								    """'s""",
								    """$""",
								    """5""",
								    """@,@""",
								    """000""",
								    """with""",
								    """3""",
								    """@.@""",
								    """34""",
								    """m""",
								    """.""",
								    """What""",
								    """'s""",
								    """up""",
								    """!""",
								    """?""",
								]

								self.assertListEqual(tokenizer.tokenize(__magic_name__							)						, __magic_name__							)

								self.assertEqual(tokenizer.convert_tokens_to_string(__magic_name__							)						, __magic_name__							)





							def  lowerCamelCase__				(		self		:List[Any]							):







								'''simple docstring'''







								a									=				self.get_tokenizer()
								a									=				len(__magic_name__							)

								tokenizer.add_tokens(["""new1""", """new2"""]							)
								tokenizer.move_added_token("""new1"""						, 1							)

								# Check that moved token is not copied (duplicate)
								self.assertEqual(len(__magic_name__							)						, original_len + 2							)
								# Check that token is moved to specified id
								self.assertEqual(tokenizer.encode("""new1"""							)						, [1]							)
								self.assertEqual(tokenizer.decode([1]							)						, """new1"""							)

