
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings




@maybe_allow_in_graph
class        _SCREAMING_SNAKE_CASE	( nn.Module		):



	def __init__(       self		,    lowercase		,    lowercase		,    lowercase		,    lowercase=0.0		,    lowercase = None		,    lowercase = "geglu"		,    lowercase = None		,    lowercase = False		,    lowercase = False		,    lowercase = False		,    lowercase = False		,    lowercase = True		,    lowercase = "layer_norm"		,    lowercase = False		,    )							->			str:
					super().__init__()
					lowerCamelCase_         =     only_cross_attention

					lowerCamelCase_         =     (num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
					lowerCamelCase_         =     (num_embeds_ada_norm is not None) and norm_type == "ada_norm"

					if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
									raise ValueError(
									    f'`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'
									    f' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'					)

					# Define 3 blocks. Each block has its own normalization layer.
					# 1. Self-Attn
					if self.use_ada_layer_norm:
									lowerCamelCase_         =     AdaLayerNorm(lowercase		,    lowercase					)
					elif self.use_ada_layer_norm_zero:
									lowerCamelCase_         =     AdaLayerNormZero(lowercase		,    lowercase					)
					else:
									lowerCamelCase_         =     nn.LayerNorm(lowercase		,    elementwise_affine=lowercase					)
					lowerCamelCase_         =     Attention(
					    query_dim=lowercase		,    heads=lowercase		,    dim_head=lowercase		,    dropout=lowercase		,    bias=lowercase		,    cross_attention_dim=cross_attention_dim if only_cross_attention else None		,    upcast_attention=lowercase		,    )

					# 2. Cross-Attn
					if cross_attention_dim is not None or double_self_attention:
									# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
									# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
									# the second cross attention block.
									lowerCamelCase_         =     (
									    AdaLayerNorm(lowercase		,    lowercase					)
									    if self.use_ada_layer_norm
									    else nn.LayerNorm(lowercase		,    elementwise_affine=lowercase					)
									)
									lowerCamelCase_         =     Attention(
									    query_dim=lowercase		,    cross_attention_dim=cross_attention_dim if not double_self_attention else None		,    heads=lowercase		,    dim_head=lowercase		,    dropout=lowercase		,    bias=lowercase		,    upcast_attention=lowercase		,    )  # is self-attn if encoder_hidden_states is none
					else:
									lowerCamelCase_         =     None
									lowerCamelCase_         =     None

					# 3. Feed-forward
					lowerCamelCase_         =     nn.LayerNorm(lowercase		,    elementwise_affine=lowercase					)
					lowerCamelCase_         =     FeedForward(lowercase		,    dropout=lowercase		,    activation_fn=lowercase		,    final_dropout=lowercase					)

					# let chunk size default to None
					lowerCamelCase_         =     None
					lowerCamelCase_         =     0



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase					)							->			List[Any]:
					# Sets chunk feed-forward
					lowerCamelCase_         =     chunk_size
					lowerCamelCase_         =     dim



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase = None		,    lowercase = None		,    lowercase = None		,    lowercase = None		,    lowercase = None		,    lowercase = None		,    )							->			Optional[int]:
					# Notice that normalization is always applied before the real computation in the following blocks.
					# 1. Self-Attention
					if self.use_ada_layer_norm:
									lowerCamelCase_         =     self.norma(lowercase		,    lowercase					)
					elif self.use_ada_layer_norm_zero:
									lowerCamelCase_      ,       lowerCamelCase_      ,       lowerCamelCase_      ,       lowerCamelCase_      ,       lowerCamelCase_         =     self.norma(
									    lowercase		,    lowercase		,    lowercase		,    hidden_dtype=hidden_states.dtype					)
					else:
									lowerCamelCase_         =     self.norma(lowercase					)

					lowerCamelCase_         =     cross_attention_kwargs if cross_attention_kwargs is not None else {}

					lowerCamelCase_         =     self.attna(
					    lowercase		,    encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None		,    attention_mask=lowercase		,    **lowercase		,    )
					if self.use_ada_layer_norm_zero:
									lowerCamelCase_         =     gate_msa.unsqueeze(1					) * attn_output
					lowerCamelCase_         =     attn_output + hidden_states

					# 2. Cross-Attention
					if self.attna is not None:
									lowerCamelCase_         =     (
									    self.norma(lowercase		,    lowercase					) if self.use_ada_layer_norm else self.norma(lowercase					)
									)

									lowerCamelCase_         =     self.attna(
									    lowercase		,    encoder_hidden_states=lowercase		,    attention_mask=lowercase		,    **lowercase		,    )
									lowerCamelCase_         =     attn_output + hidden_states

					# 3. Feed-forward
					lowerCamelCase_         =     self.norma(lowercase					)

					if self.use_ada_layer_norm_zero:
									lowerCamelCase_         =     norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

					if self._chunk_size is not None:
									# "feed_forward_chunk_size" can be used to save memory
									if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
													raise ValueError(
													    f'`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'					)

									lowerCamelCase_         =     norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
									lowerCamelCase_         =     torch.cat(
									    [self.ff(lowercase					) for hid_slice in norm_hidden_states.chunk(lowercase		,    dim=self._chunk_dim					)]		,    dim=self._chunk_dim		,    )
					else:
									lowerCamelCase_         =     self.ff(lowercase					)

					if self.use_ada_layer_norm_zero:
									lowerCamelCase_         =     gate_mlp.unsqueeze(1					) * ff_output

					lowerCamelCase_         =     ff_output + hidden_states

					return hidden_states




class        _SCREAMING_SNAKE_CASE	( nn.Module		):



	def __init__(       self		,    lowercase		,    lowercase = None		,    lowercase = 4		,    lowercase = 0.0		,    lowercase = "geglu"		,    lowercase = False		,    )							->			Optional[int]:
					super().__init__()
					lowerCamelCase_         =     int(dim * mult					)
					lowerCamelCase_         =     dim_out if dim_out is not None else dim

					if activation_fn == "gelu":
									lowerCamelCase_         =     GELU(lowercase		,    lowercase					)
					if activation_fn == "gelu-approximate":
									lowerCamelCase_         =     GELU(lowercase		,    lowercase		,    approximate="tanh"					)
					elif activation_fn == "geglu":
									lowerCamelCase_         =     GEGLU(lowercase		,    lowercase					)
					elif activation_fn == "geglu-approximate":
									lowerCamelCase_         =     ApproximateGELU(lowercase		,    lowercase					)

					lowerCamelCase_         =     nn.ModuleList([]					)
					# project in
					self.net.append(lowercase					)
					# project dropout
					self.net.append(nn.Dropout(lowercase					)					)
					# project out
					self.net.append(nn.Linear(lowercase		,    lowercase					)					)
					# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
					if final_dropout:
									self.net.append(nn.Dropout(lowercase					)					)



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase					)							->			Optional[Any]:
					for module in self.net:
									lowerCamelCase_         =     module(lowercase					)
					return hidden_states




class        _SCREAMING_SNAKE_CASE	( nn.Module		):



	def __init__(       self		,    lowercase		,    lowercase		,    lowercase = "none"					)							->			Tuple:
					super().__init__()
					lowerCamelCase_         =     nn.Linear(lowercase		,    lowercase					)
					lowerCamelCase_         =     approximate



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase					)							->			str:
					if gate.device.type != "mps":
									return F.gelu(lowercase		,    approximate=self.approximate					)
					# mps: gelu is not implemented for float16
					return F.gelu(gate.to(dtype=torch.floataa					)		,    approximate=self.approximate					).to(dtype=gate.dtype					)



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase					)							->			List[Any]:
					lowerCamelCase_         =     self.proj(lowercase					)
					lowerCamelCase_         =     self.gelu(lowercase					)
					return hidden_states




class        _SCREAMING_SNAKE_CASE	( nn.Module		):



	def __init__(       self		,    lowercase		,    lowercase					)							->			Union[str, Any]:
					super().__init__()
					lowerCamelCase_         =     nn.Linear(lowercase		,    dim_out * 2					)



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase					)							->			Any:
					if gate.device.type != "mps":
									return F.gelu(lowercase					)
					# mps: gelu is not implemented for float16
					return F.gelu(gate.to(dtype=torch.floataa					)					).to(dtype=gate.dtype					)



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase					)							->			Any:
					lowerCamelCase_      ,       lowerCamelCase_         =     self.proj(lowercase					).chunk(2		,    dim=-1					)
					return hidden_states * self.gelu(lowercase					)




class        _SCREAMING_SNAKE_CASE	( nn.Module		):



	def __init__(       self		,    lowercase		,    lowercase					)							->			List[Any]:
					super().__init__()
					lowerCamelCase_         =     nn.Linear(lowercase		,    lowercase					)



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase					)							->			Any:
					lowerCamelCase_         =     self.proj(lowercase					)
					return x * torch.sigmoid(1.7_0_2 * x					)




class        _SCREAMING_SNAKE_CASE	( nn.Module		):



	def __init__(       self		,    lowercase		,    lowercase					)							->			List[Any]:
					super().__init__()
					lowerCamelCase_         =     nn.Embedding(lowercase		,    lowercase					)
					lowerCamelCase_         =     nn.SiLU()
					lowerCamelCase_         =     nn.Linear(lowercase		,    embedding_dim * 2					)
					lowerCamelCase_         =     nn.LayerNorm(lowercase		,    elementwise_affine=lowercase					)



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase					)							->			Dict:
					lowerCamelCase_         =     self.linear(self.silu(self.emb(lowercase					)					)					)
					lowerCamelCase_      ,       lowerCamelCase_         =     torch.chunk(lowercase		,    2					)
					lowerCamelCase_         =     self.norm(lowercase					) * (1 + scale) + shift
					return x




class        _SCREAMING_SNAKE_CASE	( nn.Module		):



	def __init__(       self		,    lowercase		,    lowercase					)							->			Union[str, Any]:
					super().__init__()

					lowerCamelCase_         =     CombinedTimestepLabelEmbeddings(lowercase		,    lowercase					)

					lowerCamelCase_         =     nn.SiLU()
					lowerCamelCase_         =     nn.Linear(lowercase		,    6 * embedding_dim		,    bias=lowercase					)
					lowerCamelCase_         =     nn.LayerNorm(lowercase		,    elementwise_affine=lowercase		,    eps=1e-6					)



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase		,    lowercase		,    lowercase=None					)							->			Optional[int]:
					lowerCamelCase_         =     self.linear(self.silu(self.emb(lowercase		,    lowercase		,    hidden_dtype=lowercase					)					)					)
					lowerCamelCase_      ,       lowerCamelCase_      ,       lowerCamelCase_      ,       lowerCamelCase_      ,       lowerCamelCase_      ,       lowerCamelCase_         =     emb.chunk(6		,    dim=1					)
					lowerCamelCase_         =     self.norm(lowercase					) * (1 + scale_msa[:, None]) + shift_msa[:, None]
					return x, gate_msa, shift_mlp, scale_mlp, gate_mlp




class        _SCREAMING_SNAKE_CASE	( nn.Module		):



	def __init__(       self		,    lowercase		,    lowercase		,    lowercase		,    lowercase = None		,    lowercase = 1e-5					)							->			Tuple:
					super().__init__()
					lowerCamelCase_         =     num_groups
					lowerCamelCase_         =     eps

					if act_fn is None:
									lowerCamelCase_         =     None
					else:
									lowerCamelCase_         =     get_activation(lowercase					)

					lowerCamelCase_         =     nn.Linear(lowercase		,    out_dim * 2					)



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase					)							->			List[str]:
					if self.act:
									lowerCamelCase_         =     self.act(lowercase					)
					lowerCamelCase_         =     self.linear(lowercase					)
					lowerCamelCase_         =     emb[:, :, None, None]
					lowerCamelCase_      ,       lowerCamelCase_         =     emb.chunk(2		,    dim=1					)

					lowerCamelCase_         =     F.group_norm(lowercase		,    self.num_groups		,    eps=self.eps					)
					lowerCamelCase_         =     x * (1 + scale) + shift
					return x




import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__A							=logging.getLogger(__name__)





def      lowerCamelCase_					(							):
				lowerCamelCase_         =     argparse.ArgumentParser(
				    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."       )
				parser.add_argument(
				    "--dataset_name"      ,					type=lowerCamelCase__      ,					default="wikitext"      ,					help="Name of the training. Explore datasets at: hf.co/datasets."      ,					)
				parser.add_argument(
				    "--dataset_config"      ,					type=lowerCamelCase__      ,					default="wikitext-103-raw-v1"      ,					help="Configuration name of the dataset."       )
				parser.add_argument(
				    "--tokenizer_name_or_path"      ,					type=lowerCamelCase__      ,					default="sayakpaul/unigram-tokenizer-wikitext"      ,					help="Tokenizer identifier. Can be a local filepath or a Hub identifier."      ,					)
				parser.add_argument(
				    "--shard_size"      ,					type=lowerCamelCase__      ,					default=1_0_0_0      ,					help="Number of entries to go in a single shard."      ,					)
				parser.add_argument("--split"      ,					type=lowerCamelCase__      ,					default="train"      ,					choices=["train", "test", "validation"]       )
				parser.add_argument(
				    "--limit"      ,					default=lowerCamelCase__      ,					type=lowerCamelCase__      ,					help="Limit the number of shards (used for debugging)."      ,					)
				parser.add_argument(
				    "--max_length"      ,					type=lowerCamelCase__      ,					default=5_1_2      ,					help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
				    " sequence length that is a multiple of 8."      ,					)
				parser.add_argument(
				    "--output_dir"      ,					default="tf-tpu"      ,					type=lowerCamelCase__      ,					help="Output directory where the TFRecord shards will be saved. If the"
				    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
				    " shards will be directly saved to a Google Cloud Storage bucket."      ,					)

				lowerCamelCase_         =     parser.parse_args()
				return args





def      lowerCamelCase_					(							lowerCamelCase__       ):
				def fn(lowerCamelCase__       ):
								return tokenizer(examples["text"]       )

				return fn





def      lowerCamelCase_					(							lowerCamelCase__       ):
				lowerCamelCase_         =     []
				for i in range(len(tokenized_data["input_ids"]       )       ):
								lowerCamelCase_         =     {
								    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]       )       ),
								    "attention_mask": tf.train.Feature(
								        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]       )       ),
								}
								lowerCamelCase_         =     tf.train.Features(feature=lowerCamelCase__       )
								lowerCamelCase_         =     tf.train.Example(features=lowerCamelCase__       )
								lowerCamelCase_         =     example.SerializeToString()
								records.append(lowerCamelCase__       )
				return records





def      lowerCamelCase_					(							lowerCamelCase__       ):
				lowerCamelCase_         =     datasets.load_dataset(args.dataset_name      ,					args.dataset_config      ,					split=args.split       )

				if args.limit is not None:
								lowerCamelCase_         =     min(len(lowerCamelCase__       )      ,					args.limit       )
								lowerCamelCase_         =     dataset.select(range(lowerCamelCase__       )       )
								print(F'Limiting the dataset to {args.limit} entries.'       )

				lowerCamelCase_         =     AutoTokenizer.from_pretrained(args.tokenizer_name_or_path       )

				# Handle output directory creation.
				# For serializing into a Google Cloud Storage Bucket, one needs to first
				# create a bucket.
				if "gs" not in args.output_dir:
								if not os.path.exists(args.output_dir       ):
												os.makedirs(args.output_dir       )
								lowerCamelCase_         =     os.path.join(args.output_dir      ,					args.split       )
								if not os.path.exists(lowerCamelCase__       ):
												os.makedirs(lowerCamelCase__       )
				else:
								lowerCamelCase_         =     os.path.join(args.output_dir      ,					args.split       )

				# Tokenize the whole dataset at once.
				lowerCamelCase_         =     tokenize_function(lowerCamelCase__       )
				lowerCamelCase_         =     dataset.map(lowerCamelCase__      ,					batched=lowerCamelCase__      ,					num_proc=4      ,					remove_columns=["text"]       )

				# We need to concatenate all our texts together, and then split the result
				# into chunks of a fixed size, which we will call block_size. To do this, we
				# will use the map method again, with the option batched=True. When we use batched=True,
				# the function we pass to map() will be passed multiple inputs at once, allowing us
				# to group them into more or fewer examples than we had in the input.
				# This allows us to create our new fixed-length samples. The advantage of this
				# method is that we don't lose a whole lot of content from the dataset compared to the
				# case where we simply tokenize with a pre-defined max_length.

				def group_texts(lowerCamelCase__       ):
								# Concatenate all texts.
								lowerCamelCase_         =     {k: sum(examples[k]      ,					[]       ) for k in examples.keys()}
								lowerCamelCase_         =     len(concatenated_examples[list(examples.keys()       )[0]]       )
								# We drop the small remainder, though you could add padding instead if the model supports it
								# In this, as in all things, we advise you to follow your heart 🫀
								lowerCamelCase_         =     (total_length // args.max_length) * args.max_length
								# Split by chunks of max_len.
								lowerCamelCase_         =     {
								    k: [t[i : i + args.max_length] for i in range(0      ,					lowerCamelCase__      ,					args.max_length       )]
								    for k, t in concatenated_examples.items()
								}
								return result

				lowerCamelCase_         =     dataset_tokenized.map(lowerCamelCase__      ,					batched=lowerCamelCase__      ,					batch_size=1_0_0_0      ,					num_proc=4       )

				lowerCamelCase_         =     0
				lowerCamelCase_         =     0
				for shard in range(0      ,					len(lowerCamelCase__       )      ,					args.shard_size       ):
								lowerCamelCase_         =     grouped_dataset[shard : shard + args.shard_size]
								lowerCamelCase_         =     len(dataset_snapshot["input_ids"]       )
								lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					F'dataset-{shard_count}-{records_containing}.tfrecord'       )
								lowerCamelCase_         =     get_serialized_examples(lowerCamelCase__       )

								with tf.io.TFRecordWriter(lowerCamelCase__       ) as out_file:
												for i in range(len(lowerCamelCase__       )       ):
																lowerCamelCase_         =     serialized_examples[i]
																out_file.write(lowerCamelCase__       )
												print("Wrote file {} containing {} records".format(lowerCamelCase__      ,					lowerCamelCase__       )       )

								shard_count += 1
								total_records += records_containing

				with open(F'split-{args.split}-records-count.txt'      ,					"w"       ) as f:
								print(F'Total {args.split} records: {total_records}'      ,					file=lowerCamelCase__       )


if __name__ == "__main__":
			__A							=parse_args()
			main(args)


