






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCamelCase__       		=  logging.getLogger(__name__)






def 	UpperCamelCase	(							):




     '''simple docstring'''






     __snake_case      :Any 							=    argparse.ArgumentParser(
         description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""	)
     parser.add_argument(
         """--dataset_name"""						,type=UpperCamelCase__						,default="""wikitext"""						,help="""Name of the training. Explore datasets at: hf.co/datasets."""						,)
     parser.add_argument(
         """--dataset_config"""						,type=UpperCamelCase__						,default="""wikitext-103-raw-v1"""						,help="""Configuration name of the dataset."""	)
     parser.add_argument(
         """--tokenizer_name_or_path"""						,type=UpperCamelCase__						,default="""sayakpaul/unigram-tokenizer-wikitext"""						,help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""						,)
     parser.add_argument(
         """--shard_size"""						,type=UpperCamelCase__						,default=1000						,help="""Number of entries to go in a single shard."""						,)
     parser.add_argument("""--split"""						,type=UpperCamelCase__						,default="""train"""						,choices=["""train""", """test""", """validation"""]	)
     parser.add_argument(
         """--limit"""						,default=UpperCamelCase__						,type=UpperCamelCase__						,help="""Limit the number of shards (used for debugging)."""						,)
     parser.add_argument(
         """--max_length"""						,type=UpperCamelCase__						,default=512						,help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
         """ sequence length that is a multiple of 8."""						,)
     parser.add_argument(
         """--output_dir"""						,default="""tf-tpu"""						,type=UpperCamelCase__						,help="""Output directory where the TFRecord shards will be saved. If the"""
         """ path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord"""
         """ shards will be directly saved to a Google Cloud Storage bucket."""						,)

     __snake_case      :List[str] 							=    parser.parse_args()
     return args






def 	UpperCamelCase	(							snake_case__  : int	):




     '''simple docstring'''






     def fn(snake_case__  : Optional[Any]	):
          return tokenizer(examples["""text"""]	)

     return fn






def 	UpperCamelCase	(							snake_case__  : List[Any]	):




     '''simple docstring'''






     __snake_case      :Any 							=    []
     for i in range(len(tokenized_data["""input_ids"""]	)	):
          __snake_case      :List[Any] 							=    {
              '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]	)	),
              '''attention_mask''': tf.train.Feature(
                  intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]	)	),
          }
          __snake_case      :List[Any] 							=    tf.train.Features(feature=UpperCamelCase__	)
          __snake_case      :int 							=    tf.train.Example(features=UpperCamelCase__	)
          __snake_case      :List[str] 							=    example.SerializeToString()
          records.append(UpperCamelCase__	)
     return records






def 	UpperCamelCase	(							snake_case__  : Optional[Any]	):




     '''simple docstring'''






     __snake_case      :int 							=    datasets.load_dataset(args.dataset_name						,args.dataset_config						,split=args.split	)

     if args.limit is not None:
          __snake_case      :Dict 							=    min(len(UpperCamelCase__	)						,args.limit	)
          __snake_case      :int 							=    dataset.select(range(UpperCamelCase__	)	)
          print(f'''Limiting the dataset to {args.limit} entries.'''	)

     __snake_case      :Optional[Any] 							=    AutoTokenizer.from_pretrained(args.tokenizer_name_or_path	)

     # Handle output directory creation.
     # For serializing into a Google Cloud Storage Bucket, one needs to first
     # create a bucket.
     if "gs" not in args.output_dir:
          if not os.path.exists(args.output_dir	):
               os.makedirs(args.output_dir	)
          __snake_case      :Optional[int] 							=    os.path.join(args.output_dir						,args.split	)
          if not os.path.exists(UpperCamelCase__	):
               os.makedirs(UpperCamelCase__	)
     else:
          __snake_case      :List[str] 							=    os.path.join(args.output_dir						,args.split	)

     # Tokenize the whole dataset at once.
     __snake_case      :str 							=    tokenize_function(UpperCamelCase__	)
     __snake_case      :List[Any] 							=    dataset.map(UpperCamelCase__						,batched=UpperCamelCase__						,num_proc=4						,remove_columns=["""text"""]	)

     # We need to concatenate all our texts together, and then split the result
     # into chunks of a fixed size, which we will call block_size. To do this, we
     # will use the map method again, with the option batched=True. When we use batched=True,
     # the function we pass to map() will be passed multiple inputs at once, allowing us
     # to group them into more or fewer examples than we had in the input.
     # This allows us to create our new fixed-length samples. The advantage of this
     # method is that we don't lose a whole lot of content from the dataset compared to the
     # case where we simply tokenize with a pre-defined max_length.

     def group_texts(snake_case__  : Tuple	):
          # Concatenate all texts.
          __snake_case      :Any 							=    {k: sum(examples[k]						,[]	) for k in examples.keys()}
          __snake_case      :Dict 							=    len(concatenated_examples[list(examples.keys()	)[0]]	)
          # We drop the small remainder, though you could add padding instead if the model supports it
          # In this, as in all things, we advise you to follow your heart 🫀
          __snake_case      :List[Any] 							=    (total_length // args.max_length) * args.max_length
          # Split by chunks of max_len.
          __snake_case      :Tuple 							=    {
              k: [t[i : i + args.max_length] for i in range(0						,UpperCamelCase__						,args.max_length	)]
              for k, t in concatenated_examples.items()
          }
          return result

     __snake_case      :Optional[Any] 							=    dataset_tokenized.map(UpperCamelCase__						,batched=UpperCamelCase__						,batch_size=1000						,num_proc=4	)

     __snake_case      :Optional[int] 							=    0
     __snake_case      :Optional[int] 							=    0
     for shard in range(0						,len(UpperCamelCase__	)						,args.shard_size	):
          __snake_case      :str 							=    grouped_dataset[shard : shard + args.shard_size]
          __snake_case      :Dict 							=    len(dataset_snapshot["""input_ids"""]	)
          __snake_case      :Dict 							=    os.path.join(UpperCamelCase__						,f'''dataset-{shard_count}-{records_containing}.tfrecord'''	)
          __snake_case      :str 							=    get_serialized_examples(UpperCamelCase__	)

          with tf.io.TFRecordWriter(UpperCamelCase__	) as out_file:
               for i in range(len(UpperCamelCase__	)	):
                    __snake_case      :Tuple 							=    serialized_examples[i]
                    out_file.write(UpperCamelCase__	)
               print("""Wrote file {} containing {} records""".format(UpperCamelCase__						,UpperCamelCase__	)	)

          shard_count += 1
          total_records += records_containing

     with open(f'''split-{args.split}-records-count.txt'''						,"""w"""	) as f:
          print(f'''Total {args.split} records: {total_records}'''						,file=UpperCamelCase__	)


if __name__ == "__main__":
     lowerCamelCase__       		=  parse_args()
     main(args)









def 		lowerCAmelCase  (	UpperCamelCase__   :   int       ,			UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float					)      ->    float:

						"""simple docstring"""





						return round(float(moles / volume					) * nfactor					)






def 		lowerCAmelCase  (	UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float					)      ->    float:

						"""simple docstring"""





						return round(float((moles * 0.08_21 * temperature) / (volume)					)					)






def 		lowerCAmelCase  (	UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float					)      ->    float:

						"""simple docstring"""





						return round(float((moles * 0.08_21 * temperature) / (pressure)					)					)






def 		lowerCAmelCase  (	UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float					)      ->    float:

						"""simple docstring"""





						return round(float((pressure * volume) / (0.08_21 * moles)					)					)


if __name__ == "__main__":
							import doctest

							doctest.testmod()



