






"""simple docstring"""
import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


snake_case : Optional[Any]     		=		logging.getLogger(__name__)


def 							A						(				)    ->		Optional[int]:

       """simple docstring"""





       __magic_name__      				=    argparse.ArgumentParser(
           description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'  )
       parser.add_argument(
           '--dataset_name' ,     type=__snake_case ,     default='wikitext' ,     help='Name of the training. Explore datasets at: hf.co/datasets.' ,     )
       parser.add_argument(
           '--dataset_config' ,     type=__snake_case ,     default='wikitext-103-raw-v1' ,     help='Configuration name of the dataset.'  )
       parser.add_argument(
           '--tokenizer_name_or_path' ,     type=__snake_case ,     default='sayakpaul/unigram-tokenizer-wikitext' ,     help='Tokenizer identifier. Can be a local filepath or a Hub identifier.' ,     )
       parser.add_argument(
           '--shard_size' ,     type=__snake_case ,     default=1_0_0_0 ,     help='Number of entries to go in a single shard.' ,     )
       parser.add_argument('--split' ,     type=__snake_case ,     default='train' ,     choices=['train', 'test', 'validation']  )
       parser.add_argument(
           '--limit' ,     default=__snake_case ,     type=__snake_case ,     help='Limit the number of shards (used for debugging).' ,     )
       parser.add_argument(
           '--max_length' ,     type=__snake_case ,     default=5_1_2 ,     help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
           ' sequence length that is a multiple of 8.' ,     )
       parser.add_argument(
           '--output_dir' ,     default='tf-tpu' ,     type=__snake_case ,     help='Output directory where the TFRecord shards will be saved. If the'
           ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
           ' shards will be directly saved to a Google Cloud Storage bucket.' ,     )

       __magic_name__      				=    parser.parse_args()
       return args


def 							A						(				__snake_case: Any  )    ->		Optional[Any]:

       """simple docstring"""





       def fn(__snake_case: int  ):
              return tokenizer(examples['text']  )

       return fn


def 							A						(				__snake_case: int  )    ->		Union[str, Any]:

       """simple docstring"""





       __magic_name__      				=    []
       for i in range(len(tokenized_data['input_ids']  )  ):
              __magic_name__      				=    {
                  'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]  )  ),
                  'attention_mask': tf.train.Feature(
                      intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]  )  ),
              }
              __magic_name__      				=    tf.train.Features(feature=__snake_case  )
              __magic_name__      				=    tf.train.Example(features=__snake_case  )
              __magic_name__      				=    example.SerializeToString()
              records.append(__snake_case  )
       return records


def 							A						(				__snake_case: Optional[int]  )    ->		List[Any]:

       """simple docstring"""





       __magic_name__      				=    datasets.load_dataset(args.dataset_name ,     args.dataset_config ,     split=args.split  )

       if args.limit is not None:
              __magic_name__      				=    min(len(__snake_case  ) ,     args.limit  )
              __magic_name__      				=    dataset.select(range(__snake_case  )  )
              print(F"""Limiting the dataset to {args.limit} entries."""  )

       __magic_name__      				=    AutoTokenizer.from_pretrained(args.tokenizer_name_or_path  )

       # Handle output directory creation.
       # For serializing into a Google Cloud Storage Bucket, one needs to first
       # create a bucket.
       if "gs" not in args.output_dir:
              if not os.path.exists(args.output_dir  ):
                     os.makedirs(args.output_dir  )
              __magic_name__      				=    os.path.join(args.output_dir ,     args.split  )
              if not os.path.exists(__snake_case  ):
                     os.makedirs(__snake_case  )
       else:
              __magic_name__      				=    os.path.join(args.output_dir ,     args.split  )

       # Tokenize the whole dataset at once.
       __magic_name__      				=    tokenize_function(__snake_case  )
       __magic_name__      				=    dataset.map(__snake_case ,     batched=__snake_case ,     num_proc=4 ,     remove_columns=['text']  )

       # We need to concatenate all our texts together, and then split the result
       # into chunks of a fixed size, which we will call block_size. To do this, we
       # will use the map method again, with the option batched=True. When we use batched=True,
       # the function we pass to map() will be passed multiple inputs at once, allowing us
       # to group them into more or fewer examples than we had in the input.
       # This allows us to create our new fixed-length samples. The advantage of this
       # method is that we don't lose a whole lot of content from the dataset compared to the
       # case where we simply tokenize with a pre-defined max_length.

       def group_texts(__snake_case: Union[str, Any]  ):
              # Concatenate all texts.
              __magic_name__      				=    {k: sum(examples[k] ,     []  ) for k in examples.keys()}
              __magic_name__      				=    len(concatenated_examples[list(examples.keys()  )[0]]  )
              # We drop the small remainder, though you could add padding instead if the model supports it
              # In this, as in all things, we advise you to follow your heart 🫀
              __magic_name__      				=    (total_length // args.max_length) * args.max_length
              # Split by chunks of max_len.
              __magic_name__      				=    {
                  k: [t[i : i + args.max_length] for i in range(0 ,     __snake_case ,     args.max_length  )]
                  for k, t in concatenated_examples.items()
              }
              return result

       __magic_name__      				=    dataset_tokenized.map(__snake_case ,     batched=__snake_case ,     batch_size=1_0_0_0 ,     num_proc=4  )

       __magic_name__      				=    0
       __magic_name__      				=    0
       for shard in range(0 ,     len(__snake_case  ) ,     args.shard_size  ):
              __magic_name__      				=    grouped_dataset[shard : shard + args.shard_size]
              __magic_name__      				=    len(dataset_snapshot['input_ids']  )
              __magic_name__      				=    os.path.join(__snake_case ,     F"""dataset-{shard_count}-{records_containing}.tfrecord"""  )
              __magic_name__      				=    get_serialized_examples(__snake_case  )

              with tf.io.TFRecordWriter(__snake_case  ) as out_file:
                     for i in range(len(__snake_case  )  ):
                            __magic_name__      				=    serialized_examples[i]
                            out_file.write(__snake_case  )
                     print('Wrote file {} containing {} records'.format(__snake_case ,     __snake_case  )  )

              shard_count += 1
              total_records += records_containing

       with open(F"""split-{args.split}-records-count.txt""" ,     'w'  ) as f:
              print(F"""Total {args.split} records: {total_records}""" ,     file=__snake_case  )


if __name__ == "__main__":
 snake_case : Any     		=		parse_args()
 main(args)







"""simple docstring"""
def 							A						(				__snake_case: int = 1_0_0_0_0_0_0  )    ->		int:

       """simple docstring"""





       __magic_name__      				=    limit + 1
       __magic_name__      				=    [0] * limit
       for first_term in range(1 ,     __snake_case  ):
              for n in range(__snake_case ,     __snake_case ,     __snake_case  ):
                     __magic_name__      				=    first_term + n / first_term
                     if common_difference % 4:  # d must be divisble by 4
                            continue
                     else:
                            common_difference /= 4
                            if (
                                first_term > common_difference
                                and first_term < 4 * common_difference
                            ):  # since x,y,z are positive integers
                                   frequency[n] += 1  # so z>0 and a>d ,also 4d<a

       __magic_name__      				=    sum(1 for x in frequency[1:limit] if x == 1_0  )

       return count


if __name__ == "__main__":
 print(f"""{solution() = }""")