




'''simple docstring'''







import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


A									=logging.getLogger(__name__)





def 		snake_case_							():
				UpperCAmelCase     =	argparse.ArgumentParser(
				    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''  )
				parser.add_argument(
				    '''--dataset_name'''					,     type=_a					,     default='''wikitext'''					,     help='''Name of the training. Explore datasets at: hf.co/datasets.'''					,     )
				parser.add_argument(
				    '''--dataset_config'''					,     type=_a					,     default='''wikitext-103-raw-v1'''					,     help='''Configuration name of the dataset.'''  )
				parser.add_argument(
				    '''--tokenizer_name_or_path'''					,     type=_a					,     default='''sayakpaul/unigram-tokenizer-wikitext'''					,     help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''					,     )
				parser.add_argument(
				    '''--shard_size'''					,     type=_a					,     default=1_0_0_0					,     help='''Number of entries to go in a single shard.'''					,     )
				parser.add_argument('''--split'''					,     type=_a					,     default='''train'''					,     choices=['''train''', '''test''', '''validation''']  )
				parser.add_argument(
				    '''--limit'''					,     default=_a					,     type=_a					,     help='''Limit the number of shards (used for debugging).'''					,     )
				parser.add_argument(
				    '''--max_length'''					,     type=_a					,     default=5_1_2					,     help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
				    ''' sequence length that is a multiple of 8.'''					,     )
				parser.add_argument(
				    '''--output_dir'''					,     default='''tf-tpu'''					,     type=_a					,     help='''Output directory where the TFRecord shards will be saved. If the'''
				    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
				    ''' shards will be directly saved to a Google Cloud Storage bucket.'''					,     )

				UpperCAmelCase     =	parser.parse_args()
				return args





def 		snake_case_							(_a			:							Dict  ):
				def fn(_a			:							Tuple  ):
								return tokenizer(examples['''text''']  )

				return fn





def 		snake_case_							(_a			:							str  ):
				UpperCAmelCase     =	[]
				for i in range(len(tokenized_data['''input_ids''']  )  ):
								UpperCAmelCase     =	{
								    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]  )  ),
								    '''attention_mask''': tf.train.Feature(
								        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]  )  ),
								}
								UpperCAmelCase     =	tf.train.Features(feature=_a  )
								UpperCAmelCase     =	tf.train.Example(features=_a  )
								UpperCAmelCase     =	example.SerializeToString()
								records.append(_a  )
				return records





def 		snake_case_							(_a			:							List[Any]  ):
				UpperCAmelCase     =	datasets.load_dataset(args.dataset_name					,     args.dataset_config					,     split=args.split  )

				if args.limit is not None:
								UpperCAmelCase     =	min(len(_a  )					,     args.limit  )
								UpperCAmelCase     =	dataset.select(range(_a  )  )
								print(F"Limiting the dataset to {args.limit} entries."  )

				UpperCAmelCase     =	AutoTokenizer.from_pretrained(args.tokenizer_name_or_path  )

				# Handle output directory creation.
				# For serializing into a Google Cloud Storage Bucket, one needs to first
				# create a bucket.
				if "gs" not in args.output_dir:
								if not os.path.exists(args.output_dir  ):
												os.makedirs(args.output_dir  )
								UpperCAmelCase     =	os.path.join(args.output_dir					,     args.split  )
								if not os.path.exists(_a  ):
												os.makedirs(_a  )
				else:
								UpperCAmelCase     =	os.path.join(args.output_dir					,     args.split  )

				# Tokenize the whole dataset at once.
				UpperCAmelCase     =	tokenize_function(_a  )
				UpperCAmelCase     =	dataset.map(_a					,     batched=_a					,     num_proc=4					,     remove_columns=['''text''']  )

				# We need to concatenate all our texts together, and then split the result
				# into chunks of a fixed size, which we will call block_size. To do this, we
				# will use the map method again, with the option batched=True. When we use batched=True,
				# the function we pass to map() will be passed multiple inputs at once, allowing us
				# to group them into more or fewer examples than we had in the input.
				# This allows us to create our new fixed-length samples. The advantage of this
				# method is that we don't lose a whole lot of content from the dataset compared to the
				# case where we simply tokenize with a pre-defined max_length.

				def group_texts(_a			:							Dict  ):
								# Concatenate all texts.
								UpperCAmelCase     =	{k: sum(examples[k]					,     []  ) for k in examples.keys()}
								UpperCAmelCase     =	len(concatenated_examples[list(examples.keys()  )[0]]  )
								# We drop the small remainder, though you could add padding instead if the model supports it
								# In this, as in all things, we advise you to follow your heart 🫀
								UpperCAmelCase     =	(total_length // args.max_length) * args.max_length
								# Split by chunks of max_len.
								UpperCAmelCase     =	{
								    k: [t[i : i + args.max_length] for i in range(0					,     _a					,     args.max_length  )]
								    for k, t in concatenated_examples.items()
								}
								return result

				UpperCAmelCase     =	dataset_tokenized.map(_a					,     batched=_a					,     batch_size=1_0_0_0					,     num_proc=4  )

				UpperCAmelCase     =	0
				UpperCAmelCase     =	0
				for shard in range(0					,     len(_a  )					,     args.shard_size  ):
								UpperCAmelCase     =	grouped_dataset[shard : shard + args.shard_size]
								UpperCAmelCase     =	len(dataset_snapshot['''input_ids''']  )
								UpperCAmelCase     =	os.path.join(_a					,     F"dataset-{shard_count}-{records_containing}.tfrecord"  )
								UpperCAmelCase     =	get_serialized_examples(_a  )

								with tf.io.TFRecordWriter(_a  ) as out_file:
												for i in range(len(_a  )  ):
																UpperCAmelCase     =	serialized_examples[i]
																out_file.write(_a  )
												print('''Wrote file {} containing {} records'''.format(_a					,     _a  )  )

								shard_count += 1
								total_records += records_containing

				with open(F"split-{args.split}-records-count.txt"					,     '''w'''  ) as f:
								print(F"Total {args.split} records: {total_records}"					,     file=_a  )


if __name__ == "__main__":
			A									=parse_args()
			main(args)










'''simple docstring'''







import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
       OpenAIGPTConfig,
       OpenAIGPTDoubleHeadsModel,
       OpenAIGPTForSequenceClassification,
       OpenAIGPTLMHeadModel,
       OpenAIGPTModel,
   )


class     _a						:
       def __init__(			self		:			Optional[Any]     ,       lowercase		:			int     ,       lowercase		:			str=13     ,       lowercase		:			int=7     ,       lowercase		:			Optional[int]=True     ,       lowercase		:			Dict=True     ,       lowercase		:			Union[str, Any]=True     ,       lowercase		:			Union[str, Any]=99     ,       lowercase		:			Optional[Any]=32     ,       lowercase		:			List[Any]=5     ,       lowercase		:			List[Any]=4     ,       lowercase		:			int=37     ,       lowercase		:			int="gelu"     ,       lowercase		:			Tuple=0.1     ,       lowercase		:			Dict=0.1     ,       lowercase		:			Tuple=512     ,       lowercase		:			List[str]=16     ,       lowercase		:			Optional[int]=2     ,       lowercase		:			int=0.02     ,       lowercase		:			str=3     ,       lowercase		:			List[Any]=4     ,       lowercase		:			List[str]=None     ,       ):



           '''simple docstring'''

           UpperCAmelCase     =	parent
           UpperCAmelCase     =	batch_size
           UpperCAmelCase     =	seq_length
           UpperCAmelCase     =	is_training
           UpperCAmelCase     =	use_token_type_ids
           UpperCAmelCase     =	use_labels
           UpperCAmelCase     =	vocab_size
           UpperCAmelCase     =	hidden_size
           UpperCAmelCase     =	num_hidden_layers
           UpperCAmelCase     =	num_attention_heads
           UpperCAmelCase     =	intermediate_size
           UpperCAmelCase     =	hidden_act
           UpperCAmelCase     =	hidden_dropout_prob
           UpperCAmelCase     =	attention_probs_dropout_prob
           UpperCAmelCase     =	max_position_embeddings
           UpperCAmelCase     =	type_vocab_size
           UpperCAmelCase     =	type_sequence_label_size
           UpperCAmelCase     =	initializer_range
           UpperCAmelCase     =	num_labels
           UpperCAmelCase     =	num_choices
           UpperCAmelCase     =	scope
           UpperCAmelCase     =	self.vocab_size - 1
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	ids_tensor([self.batch_size, self.seq_length]     ,       self.vocab_size       )

           UpperCAmelCase     =	None
           if self.use_token_type_ids:
               UpperCAmelCase     =	ids_tensor([self.batch_size, self.seq_length]     ,       self.type_vocab_size       )

           UpperCAmelCase     =	None
           UpperCAmelCase     =	None
           UpperCAmelCase     =	None
           if self.use_labels:
               UpperCAmelCase     =	ids_tensor([self.batch_size]     ,       self.type_sequence_label_size       )
               UpperCAmelCase     =	ids_tensor([self.batch_size, self.seq_length]     ,       self.num_labels       )
               UpperCAmelCase     =	ids_tensor([self.batch_size]     ,       self.num_choices       )

           UpperCAmelCase     =	OpenAIGPTConfig(
               vocab_size=self.vocab_size     ,       n_embd=self.hidden_size     ,       n_layer=self.num_hidden_layers     ,       n_head=self.num_attention_heads     ,       n_positions=self.max_position_embeddings     ,       pad_token_id=self.pad_token_id     ,       )

           UpperCAmelCase     =	ids_tensor([self.num_hidden_layers, self.num_attention_heads]     ,       2       )

           return (
               config,
               input_ids,
               head_mask,
               token_type_ids,
               sequence_labels,
               token_labels,
               choice_labels,
           )
       def 					A   (			self		:			Any     ,       lowercase		:			Tuple     ,       lowercase		:			List[Any]     ,       lowercase		:			Optional[Any]     ,       lowercase		:			List[str]     ,       *lowercase		:			Optional[int]       ):



           '''simple docstring'''

           UpperCAmelCase     =	OpenAIGPTModel(config=lowercase       )
           model.to(lowercase       )
           model.eval()

           UpperCAmelCase     =	model(lowercase     ,       token_type_ids=lowercase     ,       head_mask=lowercase       )
           UpperCAmelCase     =	model(lowercase     ,       token_type_ids=lowercase       )
           UpperCAmelCase     =	model(lowercase       )

           self.parent.assertEqual(result.last_hidden_state.shape     ,       (self.batch_size, self.seq_length, self.hidden_size)       )
       def 					A   (			self		:			List[str]     ,       lowercase		:			Optional[int]     ,       lowercase		:			List[Any]     ,       lowercase		:			str     ,       lowercase		:			Optional[int]     ,       *lowercase		:			List[str]       ):



           '''simple docstring'''

           UpperCAmelCase     =	OpenAIGPTLMHeadModel(lowercase       )
           model.to(lowercase       )
           model.eval()

           UpperCAmelCase     =	model(lowercase     ,       token_type_ids=lowercase     ,       labels=lowercase       )
           self.parent.assertEqual(result.loss.shape     ,       ()       )
           self.parent.assertEqual(result.logits.shape     ,       (self.batch_size, self.seq_length, self.vocab_size)       )
       def 					A   (			self		:			Optional[int]     ,       lowercase		:			Tuple     ,       lowercase		:			Tuple     ,       lowercase		:			Optional[int]     ,       lowercase		:			Optional[int]     ,       *lowercase		:			Optional[int]       ):



           '''simple docstring'''

           UpperCAmelCase     =	OpenAIGPTDoubleHeadsModel(lowercase       )
           model.to(lowercase       )
           model.eval()

           UpperCAmelCase     =	model(lowercase     ,       token_type_ids=lowercase     ,       labels=lowercase       )
           self.parent.assertEqual(result.loss.shape     ,       ()       )
           self.parent.assertEqual(result.logits.shape     ,       (self.batch_size, self.seq_length, self.vocab_size)       )
       def 					A   (			self		:			Tuple     ,       lowercase		:			Dict     ,       lowercase		:			Any     ,       lowercase		:			Any     ,       lowercase		:			Optional[int]     ,       *lowercase		:			int       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.num_labels
           UpperCAmelCase     =	OpenAIGPTForSequenceClassification(lowercase       )
           model.to(lowercase       )
           model.eval()

           UpperCAmelCase     =	ids_tensor([self.batch_size]     ,       self.type_sequence_label_size       )
           UpperCAmelCase     =	model(lowercase     ,       token_type_ids=lowercase     ,       labels=lowercase       )
           self.parent.assertEqual(result.logits.shape     ,       (self.batch_size, self.num_labels)       )







       def 					A   (			self		:			List[str]       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.prepare_config_and_inputs()
           (
               (
               UpperCAmelCase
           ) ,  (
               UpperCAmelCase
           ) ,  (
               UpperCAmelCase
           ) ,  (
               UpperCAmelCase
           ) ,  (
               UpperCAmelCase
           ) ,  (
               UpperCAmelCase
           ) ,  (
               UpperCAmelCase
           ) ,  
           )     =	config_and_inputs
           UpperCAmelCase     =	{
               '''input_ids''': input_ids,
               '''token_type_ids''': token_type_ids,
               '''head_mask''': head_mask,
           }

           return config, inputs_dict






@require_torch
class     _a						(			__a							,							__a							,							__a							,							unittest.TestCase				):
       __a       :  List[str]            =      (
           (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
           if is_torch_available()
           else ()
       )
       __a       :  List[Any]            =      (
           (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
       )  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
       __a       :  Dict            =      (
           {
               """feature-extraction""": OpenAIGPTModel,
               """text-classification""": OpenAIGPTForSequenceClassification,
               """text-generation""": OpenAIGPTLMHeadModel,
               """zero-shot""": OpenAIGPTForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       def 					A   (			self		:			Optional[Any]     ,       lowercase		:			Union[str, Any]     ,       lowercase		:			Tuple     ,       lowercase		:			int     ,       lowercase		:			List[Any]     ,       lowercase		:			Optional[Any]       ):



           '''simple docstring'''

           if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
               # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
               # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
               # tiny config could not be created.
               return True

           return False
       def 					A   (			self		:			List[Any]     ,       lowercase		:			int     ,       lowercase		:			List[Any]     ,       lowercase		:			Optional[Any]=False       ):



           '''simple docstring'''

           UpperCAmelCase     =	super()._prepare_for_class(lowercase     ,       lowercase     ,       return_labels=lowercase       )

           if return_labels:
               if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
                   UpperCAmelCase     =	torch.zeros(
                       (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)     ,       dtype=torch.long     ,       device=lowercase     ,       )
                   UpperCAmelCase     =	inputs_dict['''labels''']
                   UpperCAmelCase     =	inputs_dict['''labels''']
                   UpperCAmelCase     =	torch.zeros(
                       (self.model_tester.batch_size, self.model_tester.num_choices)     ,       dtype=torch.long     ,       device=lowercase     ,       )
                   UpperCAmelCase     =	torch.zeros(
                       self.model_tester.batch_size     ,       dtype=torch.long     ,       device=lowercase       )
           return inputs_dict
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	OpenAIGPTModelTester(self       )
           UpperCAmelCase     =	ConfigTester(self     ,       config_class=lowercase     ,       n_embd=37       )
       def 					A   (			self		:			str       ):



           '''simple docstring'''

           self.config_tester.run_common_tests()
       def 					A   (			self		:			Tuple       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_openai_gpt_model(*lowercase       )
       def 					A   (			self		:			Optional[int]       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_lm_head_model(*lowercase       )
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_double_lm_head_model(*lowercase       )
       def 					A   (			self		:			Optional[int]       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*lowercase       )







       @slow
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               UpperCAmelCase     =	OpenAIGPTModel.from_pretrained(lowercase       )
               self.assertIsNotNone(lowercase       )






@require_torch
class     _a						(			unittest.TestCase				):
       @slow
       def 					A   (			self		:			Any       ):



           '''simple docstring'''

           UpperCAmelCase     =	OpenAIGPTLMHeadModel.from_pretrained('''openai-gpt'''       )
           model.to(lowercase       )
           UpperCAmelCase     =	torch.tensor([[481, 4_735, 544]]     ,       dtype=torch.long     ,       device=lowercase       )  # the president is
           UpperCAmelCase     =	[
               481,
               4_735,
               544,
               246,
               963,
               870,
               762,
               239,
               244,
               40_477,
               244,
               249,
               719,
               881,
               487,
               544,
               240,
               244,
               603,
               481,
           ]  # the president is a very good man. " \n " i\'m sure he is, " said the

           UpperCAmelCase     =	model.generate(lowercase     ,       do_sample=lowercase       )
           self.assertListEqual(output_ids[0].tolist()     ,       lowercase       )




