

'''simple docstring'''





import unittest

from transformers import EsmConfig, is_torch_available
from transformers.testing_utils import TestCasePlus, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import EsmForMaskedLM, EsmForSequenceClassification, EsmForTokenClassification, EsmModel
				from transformers.models.esm.modeling_esm import (
				    ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
				    EsmEmbeddings,
				    create_position_ids_from_input_ids,
				)


class  UpperCAmelCase :

		'''simple docstring'''






		def __init__(					self   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_=13   ,			SCREAMING_SNAKE_CASE_=7   ,			SCREAMING_SNAKE_CASE_=False   ,			SCREAMING_SNAKE_CASE_=True   ,			SCREAMING_SNAKE_CASE_=False   ,			SCREAMING_SNAKE_CASE_=True   ,			SCREAMING_SNAKE_CASE_=33   ,			SCREAMING_SNAKE_CASE_=32   ,			SCREAMING_SNAKE_CASE_=5   ,			SCREAMING_SNAKE_CASE_=4   ,			SCREAMING_SNAKE_CASE_=37   ,			SCREAMING_SNAKE_CASE_="gelu"   ,			SCREAMING_SNAKE_CASE_=0.1   ,			SCREAMING_SNAKE_CASE_=0.1   ,			SCREAMING_SNAKE_CASE_=512   ,			SCREAMING_SNAKE_CASE_=16   ,			SCREAMING_SNAKE_CASE_=2   ,			SCREAMING_SNAKE_CASE_=0.02   ,			SCREAMING_SNAKE_CASE_=3   ,			SCREAMING_SNAKE_CASE_=4   ,			SCREAMING_SNAKE_CASE_=None   ,			)  ->     int:






							'''simple docstring'''



							lowerCamelCase_  				=      parent
							lowerCamelCase_  				=      batch_size
							lowerCamelCase_  				=      seq_length
							lowerCamelCase_  				=      is_training
							lowerCamelCase_  				=      use_input_mask
							lowerCamelCase_  				=      use_token_type_ids
							lowerCamelCase_  				=      use_labels
							lowerCamelCase_  				=      vocab_size
							lowerCamelCase_  				=      hidden_size
							lowerCamelCase_  				=      num_hidden_layers
							lowerCamelCase_  				=      num_attention_heads
							lowerCamelCase_  				=      intermediate_size
							lowerCamelCase_  				=      hidden_act
							lowerCamelCase_  				=      hidden_dropout_prob
							lowerCamelCase_  				=      attention_probs_dropout_prob
							lowerCamelCase_  				=      max_position_embeddings
							lowerCamelCase_  				=      type_vocab_size
							lowerCamelCase_  				=      type_sequence_label_size
							lowerCamelCase_  				=      initializer_range
							lowerCamelCase_  				=      num_labels
							lowerCamelCase_  				=      num_choices
							lowerCamelCase_  				=      scope
		def 			UpperCamelCase(					self		)  ->     str:






							'''simple docstring'''



							lowerCamelCase_  				=      ids_tensor([self.batch_size, self.seq_length]   ,			self.vocab_size		)

							lowerCamelCase_  				=      None
							if self.use_input_mask:
												lowerCamelCase_  				=      random_attention_mask([self.batch_size, self.seq_length]		)

							lowerCamelCase_  				=      None
							lowerCamelCase_  				=      None
							lowerCamelCase_  				=      None
							if self.use_labels:
												lowerCamelCase_  				=      ids_tensor([self.batch_size]   ,			self.type_sequence_label_size		)
												lowerCamelCase_  				=      ids_tensor([self.batch_size, self.seq_length]   ,			self.num_labels		)
												lowerCamelCase_  				=      ids_tensor([self.batch_size]   ,			self.num_choices		)

							lowerCamelCase_  				=      self.get_config()

							return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
		def 			UpperCamelCase(					self		)  ->     List[str]:






							'''simple docstring'''



							return EsmConfig(
							    vocab_size=self.vocab_size   ,			hidden_size=self.hidden_size   ,			pad_token_id=1   ,			num_hidden_layers=self.num_hidden_layers   ,			num_attention_heads=self.num_attention_heads   ,			intermediate_size=self.intermediate_size   ,			hidden_act=self.hidden_act   ,			hidden_dropout_prob=self.hidden_dropout_prob   ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,			max_position_embeddings=self.max_position_embeddings   ,			type_vocab_size=self.type_vocab_size   ,			initializer_range=self.initializer_range   ,			)
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)  ->     Optional[int]:






							'''simple docstring'''



							lowerCamelCase_  				=      EsmModel(config=SCREAMING_SNAKE_CASE_		)
							model.to(SCREAMING_SNAKE_CASE_		)
							model.eval()
							lowerCamelCase_  				=      model(SCREAMING_SNAKE_CASE_   ,			attention_mask=SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      model(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      model(SCREAMING_SNAKE_CASE_		)

							self.parent.assertEqual(result.last_hidden_state.shape   ,			(self.batch_size, self.seq_length, self.hidden_size)		)
							self.parent.assertEqual(result.pooler_output.shape   ,			(self.batch_size, self.hidden_size)		)
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)  ->     Tuple:






							'''simple docstring'''



							lowerCamelCase_  				=      EsmForMaskedLM(config=SCREAMING_SNAKE_CASE_		)
							model.to(SCREAMING_SNAKE_CASE_		)
							model.eval()
							lowerCamelCase_  				=      model(SCREAMING_SNAKE_CASE_   ,			attention_mask=SCREAMING_SNAKE_CASE_   ,			labels=SCREAMING_SNAKE_CASE_		)
							self.parent.assertEqual(result.logits.shape   ,			(self.batch_size, self.seq_length, self.vocab_size)		)
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)  ->     Any:






							'''simple docstring'''



							lowerCamelCase_  				=      self.num_labels
							lowerCamelCase_  				=      EsmForTokenClassification(config=SCREAMING_SNAKE_CASE_		)
							model.to(SCREAMING_SNAKE_CASE_		)
							model.eval()
							lowerCamelCase_  				=      model(SCREAMING_SNAKE_CASE_   ,			attention_mask=SCREAMING_SNAKE_CASE_   ,			labels=SCREAMING_SNAKE_CASE_		)
							self.parent.assertEqual(result.logits.shape   ,			(self.batch_size, self.seq_length, self.num_labels)		)






		def 			UpperCamelCase(					self		)  ->     Any:






							'''simple docstring'''



							lowerCamelCase_  				=      self.prepare_config_and_inputs()
							(
							    (
							    lowerCamelCase_
							)				,(
							    lowerCamelCase_
							)				,(
							    lowerCamelCase_
							)				,(
							    lowerCamelCase_
							)				,(
							    lowerCamelCase_
							)				,(
							    lowerCamelCase_
							)				,
							)  				=      config_and_inputs
							lowerCamelCase_  				=      {'input_ids': input_ids, 'attention_mask': input_mask}
							return config, inputs_dict


@require_torch
class  UpperCAmelCase (							UpperCAmelCase__  , UpperCAmelCase__  , unittest.TestCase    ):

		'''simple docstring'''






		SCREAMING_SNAKE_CASE_           =					False

		SCREAMING_SNAKE_CASE_           =					(
		    (
		        EsmForMaskedLM,
		        EsmModel,
		        EsmForSequenceClassification,
		        EsmForTokenClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		SCREAMING_SNAKE_CASE_           =					()
		SCREAMING_SNAKE_CASE_           =					(
		    {
		        'feature-extraction': EsmModel,
		        'fill-mask': EsmForMaskedLM,
		        'text-classification': EsmForSequenceClassification,
		        'token-classification': EsmForTokenClassification,
		        'zero-shot': EsmForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		SCREAMING_SNAKE_CASE_           =					True
		def 			UpperCamelCase(					self		)  ->     List[Any]:






							'''simple docstring'''



							lowerCamelCase_  				=      EsmModelTester(self		)
							lowerCamelCase_  				=      ConfigTester(self   ,			config_class=SCREAMING_SNAKE_CASE_   ,			hidden_size=37		)
		def 			UpperCamelCase(					self		)  ->     List[Any]:






							'''simple docstring'''



							self.config_tester.run_common_tests()
		def 			UpperCamelCase(					self		)  ->     Any:






							'''simple docstring'''



							lowerCamelCase_  				=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_		)
		def 			UpperCamelCase(					self		)  ->     str:






							'''simple docstring'''



							lowerCamelCase_  				=      self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
												lowerCamelCase_  				=      type
												self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_		)
		def 			UpperCamelCase(					self		)  ->     Dict:






							'''simple docstring'''



							lowerCamelCase_  				=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE_		)
		def 			UpperCamelCase(					self		)  ->     int:






							'''simple docstring'''



							lowerCamelCase_  				=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE_		)
		@slow
		def 			UpperCamelCase(					self		)  ->     Any:






							'''simple docstring'''



							for model_name in ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												lowerCamelCase_  				=      EsmModel.from_pretrained(SCREAMING_SNAKE_CASE_		)
												self.assertIsNotNone(SCREAMING_SNAKE_CASE_		)
		def 			UpperCamelCase(					self		)  ->     Any:






							'''simple docstring'''



							lowerCamelCase_  				=      self.model_tester.prepare_config_and_inputs()[0]
							lowerCamelCase_  				=      EsmEmbeddings(config=SCREAMING_SNAKE_CASE_		)

							lowerCamelCase_  				=      torch.as_tensor([[12, 31, 13, model.padding_idx]]		)
							lowerCamelCase_  				=      torch.as_tensor(
							    [
							        [
							            0 + model.padding_idx + 1,
							            1 + model.padding_idx + 1,
							            2 + model.padding_idx + 1,
							            model.padding_idx,
							        ]
							    ]		)
							lowerCamelCase_  				=      create_position_ids_from_input_ids(SCREAMING_SNAKE_CASE_   ,			model.padding_idx		)
							self.assertEqual(position_ids.shape   ,			expected_positions.shape		)
							self.assertTrue(torch.all(torch.eq(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)		)		)
		def 			UpperCamelCase(					self		)  ->     List[Any]:






							'''simple docstring'''



							lowerCamelCase_  				=      self.model_tester.prepare_config_and_inputs()[0]
							lowerCamelCase_  				=      EsmEmbeddings(config=SCREAMING_SNAKE_CASE_		)

							lowerCamelCase_  				=      torch.empty(2   ,			4   ,			30		)
							lowerCamelCase_  				=      [
							    0 + embeddings.padding_idx + 1,
							    1 + embeddings.padding_idx + 1,
							    2 + embeddings.padding_idx + 1,
							    3 + embeddings.padding_idx + 1,
							]
							lowerCamelCase_  				=      torch.as_tensor([expected_single_positions, expected_single_positions]		)
							lowerCamelCase_  				=      embeddings.create_position_ids_from_inputs_embeds(SCREAMING_SNAKE_CASE_		)
							self.assertEqual(position_ids.shape   ,			expected_positions.shape		)
							self.assertTrue(torch.all(torch.eq(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)		)		)
		@unittest.skip('Esm does not support embedding resizing'		)
		def 			UpperCamelCase(					self		)  ->     Dict:






							'''simple docstring'''



							pass
		@unittest.skip('Esm does not support embedding resizing'		)
		def 			UpperCamelCase(					self		)  ->     Any:






							'''simple docstring'''



							pass






		@unittest.skip('Will be fixed soon by reducing the size of the model used for common tests.'		)
		def 			UpperCamelCase(					self		)  ->     List[Any]:






							'''simple docstring'''



							pass


@require_torch
class  UpperCAmelCase (							UpperCAmelCase__    ):

		'''simple docstring'''






		@slow
		def 			UpperCamelCase(					self		)  ->     Any:






							'''simple docstring'''



							with torch.no_grad():
												lowerCamelCase_  				=      EsmForMaskedLM.from_pretrained('facebook/esm2_t6_8M_UR50D'		)
												model.eval()
												lowerCamelCase_  				=      torch.tensor([[0, 1, 2, 3, 4, 5]]		)
												lowerCamelCase_  				=      model(SCREAMING_SNAKE_CASE_		)[0]

												lowerCamelCase_  				=      33

												lowerCamelCase_  				=      torch.Size((1, 6, vocab_size)		)
												self.assertEqual(output.shape   ,			SCREAMING_SNAKE_CASE_		)

												lowerCamelCase_  				=      torch.tensor(
												    [[[8.9_215, -10.5_898, -6.4_671], [-6.3_967, -13.9_114, -1.1_212], [-7.7_812, -13.9_516, -3.7_406]]]		)
												self.assertTrue(torch.allclose(output[:, :3, :3]   ,			SCREAMING_SNAKE_CASE_   ,			atol=1E-4		)		)






		@slow
		def 			UpperCamelCase(					self		)  ->     Tuple:






							'''simple docstring'''



							with torch.no_grad():
												lowerCamelCase_  				=      EsmModel.from_pretrained('facebook/esm2_t6_8M_UR50D'		)
												model.eval()

												lowerCamelCase_  				=      torch.tensor([[0, 6, 4, 13, 5, 4, 16, 12, 11, 7, 2]]		)
												lowerCamelCase_  				=      model(SCREAMING_SNAKE_CASE_		)[0]
												# compare the actual values for a slice.
												lowerCamelCase_  				=      torch.tensor(
												    [[[0.1_444, 0.5_413, 0.3_248], [0.3_034, 0.0_053, 0.3_108], [0.3_228, -0.2_499, 0.3_415]]]		)
												self.assertTrue(torch.allclose(output[:, :3, :3]   ,			SCREAMING_SNAKE_CASE_   ,			atol=1E-4		)		)





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__A							=logging.getLogger(__name__)





def      lowerCamelCase_					(							):
				lowerCamelCase_         =     argparse.ArgumentParser(
				    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."       )
				parser.add_argument(
				    "--dataset_name"      ,					type=lowerCamelCase__      ,					default="wikitext"      ,					help="Name of the training. Explore datasets at: hf.co/datasets."      ,					)
				parser.add_argument(
				    "--dataset_config"      ,					type=lowerCamelCase__      ,					default="wikitext-103-raw-v1"      ,					help="Configuration name of the dataset."       )
				parser.add_argument(
				    "--tokenizer_name_or_path"      ,					type=lowerCamelCase__      ,					default="sayakpaul/unigram-tokenizer-wikitext"      ,					help="Tokenizer identifier. Can be a local filepath or a Hub identifier."      ,					)
				parser.add_argument(
				    "--shard_size"      ,					type=lowerCamelCase__      ,					default=1_0_0_0      ,					help="Number of entries to go in a single shard."      ,					)
				parser.add_argument("--split"      ,					type=lowerCamelCase__      ,					default="train"      ,					choices=["train", "test", "validation"]       )
				parser.add_argument(
				    "--limit"      ,					default=lowerCamelCase__      ,					type=lowerCamelCase__      ,					help="Limit the number of shards (used for debugging)."      ,					)
				parser.add_argument(
				    "--max_length"      ,					type=lowerCamelCase__      ,					default=5_1_2      ,					help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
				    " sequence length that is a multiple of 8."      ,					)
				parser.add_argument(
				    "--output_dir"      ,					default="tf-tpu"      ,					type=lowerCamelCase__      ,					help="Output directory where the TFRecord shards will be saved. If the"
				    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
				    " shards will be directly saved to a Google Cloud Storage bucket."      ,					)

				lowerCamelCase_         =     parser.parse_args()
				return args





def      lowerCamelCase_					(							lowerCamelCase__       ):
				def fn(lowerCamelCase__       ):
								return tokenizer(examples["text"]       )

				return fn





def      lowerCamelCase_					(							lowerCamelCase__       ):
				lowerCamelCase_         =     []
				for i in range(len(tokenized_data["input_ids"]       )       ):
								lowerCamelCase_         =     {
								    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]       )       ),
								    "attention_mask": tf.train.Feature(
								        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]       )       ),
								}
								lowerCamelCase_         =     tf.train.Features(feature=lowerCamelCase__       )
								lowerCamelCase_         =     tf.train.Example(features=lowerCamelCase__       )
								lowerCamelCase_         =     example.SerializeToString()
								records.append(lowerCamelCase__       )
				return records





def      lowerCamelCase_					(							lowerCamelCase__       ):
				lowerCamelCase_         =     datasets.load_dataset(args.dataset_name      ,					args.dataset_config      ,					split=args.split       )

				if args.limit is not None:
								lowerCamelCase_         =     min(len(lowerCamelCase__       )      ,					args.limit       )
								lowerCamelCase_         =     dataset.select(range(lowerCamelCase__       )       )
								print(F'Limiting the dataset to {args.limit} entries.'       )

				lowerCamelCase_         =     AutoTokenizer.from_pretrained(args.tokenizer_name_or_path       )

				# Handle output directory creation.
				# For serializing into a Google Cloud Storage Bucket, one needs to first
				# create a bucket.
				if "gs" not in args.output_dir:
								if not os.path.exists(args.output_dir       ):
												os.makedirs(args.output_dir       )
								lowerCamelCase_         =     os.path.join(args.output_dir      ,					args.split       )
								if not os.path.exists(lowerCamelCase__       ):
												os.makedirs(lowerCamelCase__       )
				else:
								lowerCamelCase_         =     os.path.join(args.output_dir      ,					args.split       )

				# Tokenize the whole dataset at once.
				lowerCamelCase_         =     tokenize_function(lowerCamelCase__       )
				lowerCamelCase_         =     dataset.map(lowerCamelCase__      ,					batched=lowerCamelCase__      ,					num_proc=4      ,					remove_columns=["text"]       )

				# We need to concatenate all our texts together, and then split the result
				# into chunks of a fixed size, which we will call block_size. To do this, we
				# will use the map method again, with the option batched=True. When we use batched=True,
				# the function we pass to map() will be passed multiple inputs at once, allowing us
				# to group them into more or fewer examples than we had in the input.
				# This allows us to create our new fixed-length samples. The advantage of this
				# method is that we don't lose a whole lot of content from the dataset compared to the
				# case where we simply tokenize with a pre-defined max_length.

				def group_texts(lowerCamelCase__       ):
								# Concatenate all texts.
								lowerCamelCase_         =     {k: sum(examples[k]      ,					[]       ) for k in examples.keys()}
								lowerCamelCase_         =     len(concatenated_examples[list(examples.keys()       )[0]]       )
								# We drop the small remainder, though you could add padding instead if the model supports it
								# In this, as in all things, we advise you to follow your heart 🫀
								lowerCamelCase_         =     (total_length // args.max_length) * args.max_length
								# Split by chunks of max_len.
								lowerCamelCase_         =     {
								    k: [t[i : i + args.max_length] for i in range(0      ,					lowerCamelCase__      ,					args.max_length       )]
								    for k, t in concatenated_examples.items()
								}
								return result

				lowerCamelCase_         =     dataset_tokenized.map(lowerCamelCase__      ,					batched=lowerCamelCase__      ,					batch_size=1_0_0_0      ,					num_proc=4       )

				lowerCamelCase_         =     0
				lowerCamelCase_         =     0
				for shard in range(0      ,					len(lowerCamelCase__       )      ,					args.shard_size       ):
								lowerCamelCase_         =     grouped_dataset[shard : shard + args.shard_size]
								lowerCamelCase_         =     len(dataset_snapshot["input_ids"]       )
								lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					F'dataset-{shard_count}-{records_containing}.tfrecord'       )
								lowerCamelCase_         =     get_serialized_examples(lowerCamelCase__       )

								with tf.io.TFRecordWriter(lowerCamelCase__       ) as out_file:
												for i in range(len(lowerCamelCase__       )       ):
																lowerCamelCase_         =     serialized_examples[i]
																out_file.write(lowerCamelCase__       )
												print("Wrote file {} containing {} records".format(lowerCamelCase__      ,					lowerCamelCase__       )       )

								shard_count += 1
								total_records += records_containing

				with open(F'split-{args.split}-records-count.txt'      ,					"w"       ) as f:
								print(F'Total {args.split} records: {total_records}'      ,					file=lowerCamelCase__       )


if __name__ == "__main__":
			__A							=parse_args()
			main(args)


