





import tensorflow as tf

from ...tf_utils import shape_list
class      UpperCamelCase_						(					tf.keras.layers.Layer   ):




					def __init__(						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_=1			,  lowerCAmelCase_=False			,  **lowerCAmelCase_					)						->							Dict:
										super().__init__(**lowerCAmelCase_					)

										_snake_case													=			vocab_size
										_snake_case													=			d_embed
										_snake_case													=			d_proj

										_snake_case													=			cutoffs + [vocab_size]
										_snake_case													=			[0] + self.cutoffs
										_snake_case													=			div_val

										_snake_case													=			self.cutoffs[0]
										_snake_case													=			len(self.cutoffs					) - 1
										_snake_case													=			self.shortlist_size + self.n_clusters
										_snake_case													=			keep_order

										_snake_case													=			[]
										_snake_case													=			[]




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							Union[str, Any]:
										if self.n_clusters > 0:
															_snake_case													=			self.add_weight(
															    shape=(self.n_clusters, self.d_embed)			,  initializer='zeros'			,  trainable=lowerCAmelCase_			,  name='cluster_weight'					)
															_snake_case													=			self.add_weight(
															    shape=(self.n_clusters,)			,  initializer='zeros'			,  trainable=lowerCAmelCase_			,  name='cluster_bias'					)

										if self.div_val == 1:
															for i in range(len(self.cutoffs					)					):
																				if self.d_proj != self.d_embed:
																									_snake_case													=			self.add_weight(
																									    shape=(self.d_embed, self.d_proj)			,  initializer='zeros'			,  trainable=lowerCAmelCase_			,  name=F'''out_projs_._{i}'''			,  )
																									self.out_projs.append(lowerCAmelCase_					)
																				else:
																									self.out_projs.append(lowerCAmelCase_					)
																				_snake_case													=			self.add_weight(
																				    shape=(self.vocab_size, self.d_embed)			,  initializer='zeros'			,  trainable=lowerCAmelCase_			,  name=F'''out_layers_._{i}_._weight'''			,  )
																				_snake_case													=			self.add_weight(
																				    shape=(self.vocab_size,)			,  initializer='zeros'			,  trainable=lowerCAmelCase_			,  name=F'''out_layers_._{i}_._bias'''			,  )
																				self.out_layers.append((weight, bias)					)
										else:
															for i in range(len(self.cutoffs					)					):
																				_snake_case							,       _snake_case													=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
																				_snake_case													=			self.d_embed // (self.div_val**i)

																				_snake_case													=			self.add_weight(
																				    shape=(d_emb_i, self.d_proj)			,  initializer='zeros'			,  trainable=lowerCAmelCase_			,  name=F'''out_projs_._{i}'''					)
																				self.out_projs.append(lowerCAmelCase_					)
																				_snake_case													=			self.add_weight(
																				    shape=(r_idx - l_idx, d_emb_i)			,  initializer='zeros'			,  trainable=lowerCAmelCase_			,  name=F'''out_layers_._{i}_._weight'''			,  )
																				_snake_case													=			self.add_weight(
																				    shape=(r_idx - l_idx,)			,  initializer='zeros'			,  trainable=lowerCAmelCase_			,  name=F'''out_layers_._{i}_._bias'''			,  )
																				self.out_layers.append((weight, bias)					)
										super().build(lowerCAmelCase_					)




					@staticmethod
					def 		lowerCAmelCase     (						lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_=None					)						->							List[Any]:
										_snake_case													=			x
										if proj is not None:
															_snake_case													=			tf.einsum('ibd,ed->ibe'			,  lowerCAmelCase_			,  lowerCAmelCase_					)
										return tf.einsum('ibd,nd->ibn'			,  lowerCAmelCase_			,  lowerCAmelCase_					) + b




					@staticmethod
					def 		lowerCAmelCase     (						lowerCAmelCase_			,  lowerCAmelCase_					)						->							Optional[Any]:
										_snake_case													=			shape_list(lowerCAmelCase_					)
										_snake_case													=			tf.range(lp_size[0]			,  dtype=target.dtype					)
										_snake_case													=			tf.stack([r, target]			,  1					)
										return tf.gather_nd(lowerCAmelCase_			,  lowerCAmelCase_					)






					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_=True			,  lowerCAmelCase_=False					)						->							int:
										_snake_case													=			0
										if self.n_clusters == 0:
															_snake_case													=			self._logit(lowerCAmelCase_			,  self.out_layers[0][0]			,  self.out_layers[0][1]			,  self.out_projs[0]					)
															if target is not None:
																				_snake_case													=			tf.nn.sparse_softmax_cross_entropy_with_logits(labels=lowerCAmelCase_			,  logits=lowerCAmelCase_					)
															_snake_case													=			tf.nn.log_softmax(lowerCAmelCase_			,  axis=-1					)
										else:
															_snake_case													=			shape_list(lowerCAmelCase_					)
															_snake_case													=			[]
															_snake_case													=			tf.zeros(hidden_sizes[:2]					)
															for i in range(len(self.cutoffs					)					):
																				_snake_case							,       _snake_case													=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
																				if target is not None:
																									_snake_case													=			(target >= l_idx) & (target < r_idx)
																									_snake_case													=			tf.where(lowerCAmelCase_					)
																									_snake_case													=			tf.boolean_mask(lowerCAmelCase_			,  lowerCAmelCase_					) - l_idx

																				if self.div_val == 1:
																									_snake_case													=			self.out_layers[0][0][l_idx:r_idx]
																									_snake_case													=			self.out_layers[0][1][l_idx:r_idx]
																				else:
																									_snake_case													=			self.out_layers[i][0]
																									_snake_case													=			self.out_layers[i][1]

																				if i == 0:
																									_snake_case													=			tf.concat([cur_W, self.cluster_weight]			,  0					)
																									_snake_case													=			tf.concat([cur_b, self.cluster_bias]			,  0					)

																									_snake_case													=			self._logit(lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  self.out_projs[0]					)
																									_snake_case													=			tf.nn.log_softmax(lowerCAmelCase_					)
																									out.append(head_logprob[..., : self.cutoffs[0]]					)
																									if target is not None:
																														_snake_case													=			tf.boolean_mask(lowerCAmelCase_			,  lowerCAmelCase_					)
																														_snake_case													=			self._gather_logprob(lowerCAmelCase_			,  lowerCAmelCase_					)
																				else:
																									_snake_case													=			self._logit(lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  self.out_projs[i]					)
																									_snake_case													=			tf.nn.log_softmax(lowerCAmelCase_					)
																									_snake_case													=			self.cutoffs[0] + i - 1  # No probability for the head cluster
																									_snake_case													=			head_logprob[..., cluster_prob_idx, None] + tail_logprob
																									out.append(lowerCAmelCase_					)
																									if target is not None:
																														_snake_case													=			tf.boolean_mask(lowerCAmelCase_			,  lowerCAmelCase_					)
																														_snake_case													=			tf.boolean_mask(lowerCAmelCase_			,  lowerCAmelCase_					)
																														_snake_case													=			self._gather_logprob(lowerCAmelCase_			,  lowerCAmelCase_					)
																														cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
																				if target is not None:
																									loss += tf.scatter_nd(lowerCAmelCase_			,  -cur_logprob			,  shape_list(lowerCAmelCase_					)					)
															_snake_case													=			tf.concat(lowerCAmelCase_			,  axis=-1					)

										if target is not None:
															if return_mean:
																				_snake_case													=			tf.reduce_mean(lowerCAmelCase_					)
															# Add the training-time loss value to the layer using `self.add_loss()`.
															self.add_loss(lowerCAmelCase_					)

															# Log the loss as a metric (we could log arbitrary metrics,
															# including different metrics for training and inference.
															self.add_metric(lowerCAmelCase_			,  name=self.name			,  aggregation='mean' if return_mean else ''					)

										return out










import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


UpperCAmelCase_ 							=       [
    {"""dataset""": """wikipedia""", """config_name""": """20220301.de"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.en"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.fr"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.frr"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.it"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.simple"""},
    {"""dataset""": """snli""", """config_name""": """plain_text"""},
    {"""dataset""": """eli5""", """config_name""": """LFQA_reddit"""},
    {"""dataset""": """wiki40b""", """config_name""": """en"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.nq.compressed"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.nq.no_index"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.multiset.no_index"""},
    {"""dataset""": """natural_questions""", """config_name""": """default"""},
]





def   lowerCamelCase__       (       UpperCamelCase__     :    Dict=True  )     ->	Dict:



					'''simple docstring'''




					if with_config:
										return [
										    {
										        "testcase_name": d["dataset"] + "/" + d["config_name"],
										        "dataset": d["dataset"],
										        "config_name": d["config_name"],
										    }
										    for d in DATASETS_ON_HF_GCP
										]
					else:
										return [
										    {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
										]
@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=_lowerCamelCase   )   )
class      UpperCamelCase_						(					_lowerCamelCase   ):
					lowerCAmelCase_							=		None
					lowerCAmelCase_							=		None




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_					)						->							Union[str, Any]:
										with TemporaryDirectory() as tmp_dir:
															_snake_case													=			dataset_module_factory(lowerCAmelCase_			,  cache_dir=lowerCAmelCase_					)

															_snake_case													=			import_main_class(dataset_module.module_path			,  dataset=lowerCAmelCase_					)

															_snake_case													=			builder_cls(
															    cache_dir=lowerCAmelCase_			,  config_name=lowerCAmelCase_			,  hash=dataset_module.hash			,  )

															_snake_case													=			'/'.join(
															    [
															        HF_GCP_BASE_URL,
															        builder_instance._relative_data_dir(with_hash=lowerCAmelCase_					).replace(os.sep			,  '/'					),
															        config.DATASET_INFO_FILENAME,
															    ]					)
															_snake_case													=			cached_path(lowerCAmelCase_			,  cache_dir=lowerCAmelCase_					)
															self.assertTrue(os.path.exists(lowerCAmelCase_					)					)





@pytest.mark.integration
def   lowerCamelCase__       (       UpperCamelCase__     :    Any  )     ->	Tuple:



					'''simple docstring'''




					_snake_case													=			tmp_path_factory.mktemp('test_hf_gcp'  ) / 'test_wikipedia_simple'
					_snake_case													=			dataset_module_factory('wikipedia'    ,      cache_dir=UpperCamelCase__  )
					_snake_case													=			import_main_class(dataset_module.module_path  )
					_snake_case													=			builder_cls(
					    cache_dir=UpperCamelCase__    ,      config_name='20220301.frr'    ,      hash=dataset_module.hash    ,      )
					# use the HF cloud storage, not the original download_and_prepare that uses apache-beam
					_snake_case													=			None
					builder_instance.download_and_prepare()
					_snake_case													=			builder_instance.as_dataset()
					assert ds





@pytest.mark.integration
def   lowerCamelCase__       (       UpperCamelCase__     :    Union[str, Any]  )     ->	Union[str, Any]:



					'''simple docstring'''




					_snake_case													=			dataset_module_factory('wikipedia'    ,      cache_dir=UpperCamelCase__  )
					_snake_case													=			import_main_class(dataset_module.module_path    ,      dataset=UpperCamelCase__  )
					_snake_case													=			builder_cls(
					    cache_dir=UpperCamelCase__    ,      config_name='20220301.frr'    ,      hash=dataset_module.hash    ,      )
					_snake_case													=			builder_instance.as_streaming_dataset()
					assert ds
					assert isinstance(UpperCamelCase__    ,      UpperCamelCase__  )
					assert "train" in ds
					assert isinstance(ds['train']    ,      UpperCamelCase__  )
					assert next(iter(ds['train']  )  )



