


'''simple docstring'''





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__UpperCamelCase       :       Union[str, Any]						       =				logging.getLogger(__name__)

def 			_a  (				):







 """simple docstring"""



 UpperCamelCase__					:    Any          =   argparse.ArgumentParser(
     description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''     )
 parser.add_argument(
     '''--dataset_name'''    , type=_A    , default='''wikitext'''    , help='''Name of the training. Explore datasets at: hf.co/datasets.'''    , )
 parser.add_argument(
     '''--dataset_config'''    , type=_A    , default='''wikitext-103-raw-v1'''    , help='''Configuration name of the dataset.'''     )
 parser.add_argument(
     '''--tokenizer_name_or_path'''    , type=_A    , default='''sayakpaul/unigram-tokenizer-wikitext'''    , help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''    , )
 parser.add_argument(
     '''--shard_size'''    , type=_A    , default=1000    , help='''Number of entries to go in a single shard.'''    , )
 parser.add_argument('''--split'''    , type=_A    , default='''train'''    , choices=['''train''', '''test''', '''validation''']     )
 parser.add_argument(
     '''--limit'''    , default=_A    , type=_A    , help='''Limit the number of shards (used for debugging).'''    , )
 parser.add_argument(
     '''--max_length'''    , type=_A    , default=512    , help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
     ''' sequence length that is a multiple of 8.'''    , )
 parser.add_argument(
     '''--output_dir'''    , default='''tf-tpu'''    , type=_A    , help='''Output directory where the TFRecord shards will be saved. If the'''
     ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
     ''' shards will be directly saved to a Google Cloud Storage bucket.'''    , )

 UpperCamelCase__					:    int          =   parser.parse_args()
 return args

def 			_a  (				SCREAMING_SNAKE_CASE	:				Optional[int]     ):







 """simple docstring"""



 def fn(SCREAMING_SNAKE_CASE	:				List[str]     ):
  return tokenizer(examples['''text''']     )

 return fn

def 			_a  (				SCREAMING_SNAKE_CASE	:				List[str]     ):







 """simple docstring"""



 UpperCamelCase__					:    int          =   []
 for i in range(len(tokenized_data['''input_ids''']     )     ):
  UpperCamelCase__					:    str          =   {
      '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]     )     ),
      '''attention_mask''': tf.train.Feature(
          intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]     )     ),
  }
  UpperCamelCase__					:    Union[str, Any]          =   tf.train.Features(feature=_A     )
  UpperCamelCase__					:    Any          =   tf.train.Example(features=_A     )
  UpperCamelCase__					:    str          =   example.SerializeToString()
  records.append(_A     )
 return records

def 			_a  (				SCREAMING_SNAKE_CASE	:				Union[str, Any]     ):







 """simple docstring"""



 UpperCamelCase__					:    Tuple          =   datasets.load_dataset(args.dataset_name    , args.dataset_config    , split=args.split     )

 if args.limit is not None:
  UpperCamelCase__					:    List[str]          =   min(len(_A     )    , args.limit     )
  UpperCamelCase__					:    List[str]          =   dataset.select(range(_A     )     )
  print(F"Limiting the dataset to {args.limit} entries."     )

 UpperCamelCase__					:    Union[str, Any]          =   AutoTokenizer.from_pretrained(args.tokenizer_name_or_path     )

 # Handle output directory creation.
 # For serializing into a Google Cloud Storage Bucket, one needs to first
 # create a bucket.
 if "gs" not in args.output_dir:
  if not os.path.exists(args.output_dir     ):
   os.makedirs(args.output_dir     )
  UpperCamelCase__					:    Dict          =   os.path.join(args.output_dir    , args.split     )
  if not os.path.exists(_A     ):
   os.makedirs(_A     )
 else:
  UpperCamelCase__					:    Union[str, Any]          =   os.path.join(args.output_dir    , args.split     )

 # Tokenize the whole dataset at once.
 UpperCamelCase__					:    Tuple          =   tokenize_function(_A     )
 UpperCamelCase__					:    Optional[Any]          =   dataset.map(_A    , batched=_A    , num_proc=4    , remove_columns=['''text''']     )

 # We need to concatenate all our texts together, and then split the result
 # into chunks of a fixed size, which we will call block_size. To do this, we
 # will use the map method again, with the option batched=True. When we use batched=True,
 # the function we pass to map() will be passed multiple inputs at once, allowing us
 # to group them into more or fewer examples than we had in the input.
 # This allows us to create our new fixed-length samples. The advantage of this
 # method is that we don't lose a whole lot of content from the dataset compared to the
 # case where we simply tokenize with a pre-defined max_length.

 def group_texts(SCREAMING_SNAKE_CASE	:				str     ):
  # Concatenate all texts.
  UpperCamelCase__					:    Dict          =   {k: sum(examples[k]    , []     ) for k in examples.keys()}
  UpperCamelCase__					:    int          =   len(concatenated_examples[list(examples.keys()     )[0]]     )
  # We drop the small remainder, though you could add padding instead if the model supports it
  # In this, as in all things, we advise you to follow your heart 🫀
  UpperCamelCase__					:    int          =   (total_length // args.max_length) * args.max_length
  # Split by chunks of max_len.
  UpperCamelCase__					:    Tuple          =   {
      k: [t[i : i + args.max_length] for i in range(0    , _A    , args.max_length     )]
      for k, t in concatenated_examples.items()
  }
  return result

 UpperCamelCase__					:    Optional[Any]          =   dataset_tokenized.map(_A    , batched=_A    , batch_size=1000    , num_proc=4     )

 UpperCamelCase__					:    List[Any]          =   0
 UpperCamelCase__					:    Union[str, Any]          =   0
 for shard in range(0    , len(_A     )    , args.shard_size     ):
  UpperCamelCase__					:    Any          =   grouped_dataset[shard : shard + args.shard_size]
  UpperCamelCase__					:    Union[str, Any]          =   len(dataset_snapshot['''input_ids''']     )
  UpperCamelCase__					:    List[Any]          =   os.path.join(_A    , F"dataset-{shard_count}-{records_containing}.tfrecord"     )
  UpperCamelCase__					:    Optional[int]          =   get_serialized_examples(_A     )

  with tf.io.TFRecordWriter(_A     ) as out_file:
   for i in range(len(_A     )     ):
    UpperCamelCase__					:    Optional[Any]          =   serialized_examples[i]
    out_file.write(_A     )
   print('''Wrote file {} containing {} records'''.format(_A    , _A     )     )

  shard_count += 1
  total_records += records_containing

 with open(F"split-{args.split}-records-count.txt"    , '''w'''     ) as f:
  print(F"Total {args.split} records: {total_records}"    , file=_A     )


if __name__ == "__main__":
       __UpperCamelCase       :       str						       =				parse_args()
       main(args)






import functools

def 			_a  (				SCREAMING_SNAKE_CASE	:				str    , SCREAMING_SNAKE_CASE	:				str     ):







 """simple docstring"""



 UpperCamelCase__					:    str          =   len(SCREAMING_SNAKE_CASE     )
 UpperCamelCase__					:    Optional[Any]          =   len(SCREAMING_SNAKE_CASE     )

 @functools.cache
 def min_distance(SCREAMING_SNAKE_CASE	:				int    , SCREAMING_SNAKE_CASE	:				int     ) -> int:
  # if first word index is overflow - delete all from the second word
  if indexa >= len_worda:
   return len_worda - indexa
  # if second word index is overflow - delete all from the first word
  if indexa >= len_worda:
   return len_worda - indexa
  UpperCamelCase__					:    Tuple          =   int(worda[indexa] != worda[indexa]     )  # current letters not identical
  return min(
      1 + min_distance(indexa + 1    , SCREAMING_SNAKE_CASE     )    , 1 + min_distance(SCREAMING_SNAKE_CASE    , indexa + 1     )    , diff + min_distance(indexa + 1    , indexa + 1     )    , )

 return min_distance(0    , 0     )


if __name__ == "__main__":
       import doctest

       doctest.testmod()


