

"""simple docstring"""




import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__UpperCAmelCase =				logging.getLogger(__name__)
def   lowerCAmelCase						(					):
 '''simple docstring'''





 UpperCAmelCase__					:  List[str]					   =					argparse.ArgumentParser(
     description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""      )
 parser.add_argument(
     """--dataset_name"""							,    type=__UpperCamelCase							,    default="""wikitext"""							,    help="""Name of the training. Explore datasets at: hf.co/datasets."""							,    )
 parser.add_argument(
     """--dataset_config"""							,    type=__UpperCamelCase							,    default="""wikitext-103-raw-v1"""							,    help="""Configuration name of the dataset."""      )
 parser.add_argument(
     """--tokenizer_name_or_path"""							,    type=__UpperCamelCase							,    default="""sayakpaul/unigram-tokenizer-wikitext"""							,    help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""							,    )
 parser.add_argument(
     """--shard_size"""							,    type=__UpperCamelCase							,    default=1000							,    help="""Number of entries to go in a single shard."""							,    )
 parser.add_argument("""--split"""							,    type=__UpperCamelCase							,    default="""train"""							,    choices=["""train""", """test""", """validation"""]      )
 parser.add_argument(
     """--limit"""							,    default=__UpperCamelCase							,    type=__UpperCamelCase							,    help="""Limit the number of shards (used for debugging)."""							,    )
 parser.add_argument(
     """--max_length"""							,    type=__UpperCamelCase							,    default=512							,    help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
     """ sequence length that is a multiple of 8."""							,    )
 parser.add_argument(
     """--output_dir"""							,    default="""tf-tpu"""							,    type=__UpperCamelCase							,    help="""Output directory where the TFRecord shards will be saved. If the"""
     """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
     """ shards will be directly saved to a Google Cloud Storage bucket."""							,    )

 UpperCAmelCase__					:  Any					   =					parser.parse_args()
 return args
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 def fn(__UpperCamelCase      ):
  return tokenizer(examples["""text"""]      )

 return fn
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Dict					   =					[]
 for i in range(len(tokenized_data["""input_ids"""]      )      ):
  UpperCAmelCase__					:  Any					   =					{
      """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]      )      ),
      """attention_mask""": tf.train.Feature(
          intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]      )      ),
  }
  UpperCAmelCase__					:  Optional[int]					   =					tf.train.Features(feature=__UpperCamelCase      )
  UpperCAmelCase__					:  str					   =					tf.train.Example(features=__UpperCamelCase      )
  UpperCAmelCase__					:  Union[str, Any]					   =					example.SerializeToString()
  records.append(__UpperCamelCase      )
 return records
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Optional[int]					   =					datasets.load_dataset(args.dataset_name							,    args.dataset_config							,    split=args.split      )

 if args.limit is not None:
  UpperCAmelCase__					:  Union[str, Any]					   =					min(len(__UpperCamelCase      )							,    args.limit      )
  UpperCAmelCase__					:  Tuple					   =					dataset.select(range(__UpperCamelCase      )      )
  print(F"Limiting the dataset to {args.limit} entries."      )

 UpperCAmelCase__					:  Tuple					   =					AutoTokenizer.from_pretrained(args.tokenizer_name_or_path      )

 # Handle output directory creation.
 # For serializing into a Google Cloud Storage Bucket, one needs to first
 # create a bucket.
 if "gs" not in args.output_dir:
  if not os.path.exists(args.output_dir      ):
   os.makedirs(args.output_dir      )
  UpperCAmelCase__					:  List[Any]					   =					os.path.join(args.output_dir							,    args.split      )
  if not os.path.exists(__UpperCamelCase      ):
   os.makedirs(__UpperCamelCase      )
 else:
  UpperCAmelCase__					:  str					   =					os.path.join(args.output_dir							,    args.split      )

 # Tokenize the whole dataset at once.
 UpperCAmelCase__					:  Dict					   =					tokenize_function(__UpperCamelCase      )
 UpperCAmelCase__					:  List[Any]					   =					dataset.map(__UpperCamelCase							,    batched=__UpperCamelCase							,    num_proc=4							,    remove_columns=["""text"""]      )

 # We need to concatenate all our texts together, and then split the result
 # into chunks of a fixed size, which we will call block_size. To do this, we
 # will use the map method again, with the option batched=True. When we use batched=True,
 # the function we pass to map() will be passed multiple inputs at once, allowing us
 # to group them into more or fewer examples than we had in the input.
 # This allows us to create our new fixed-length samples. The advantage of this
 # method is that we don't lose a whole lot of content from the dataset compared to the
 # case where we simply tokenize with a pre-defined max_length.

 def group_texts(__UpperCamelCase      ):
  # Concatenate all texts.
  UpperCAmelCase__					:  List[str]					   =					{k: sum(examples[k]							,    []      ) for k in examples.keys()}
  UpperCAmelCase__					:  Optional[int]					   =					len(concatenated_examples[list(examples.keys()      )[0]]      )
  # We drop the small remainder, though you could add padding instead if the model supports it
  # In this, as in all things, we advise you to follow your heart 🫀
  UpperCAmelCase__					:  List[str]					   =					(total_length // args.max_length) * args.max_length
  # Split by chunks of max_len.
  UpperCAmelCase__					:  Tuple					   =					{
      k: [t[i : i + args.max_length] for i in range(0							,    __UpperCamelCase							,    args.max_length      )]
      for k, t in concatenated_examples.items()
  }
  return result

 UpperCAmelCase__					:  int					   =					dataset_tokenized.map(__UpperCamelCase							,    batched=__UpperCamelCase							,    batch_size=1000							,    num_proc=4      )

 UpperCAmelCase__					:  Union[str, Any]					   =					0
 UpperCAmelCase__					:  int					   =					0
 for shard in range(0							,    len(__UpperCamelCase      )							,    args.shard_size      ):
  UpperCAmelCase__					:  Union[str, Any]					   =					grouped_dataset[shard : shard + args.shard_size]
  UpperCAmelCase__					:  Optional[Any]					   =					len(dataset_snapshot["""input_ids"""]      )
  UpperCAmelCase__					:  Union[str, Any]					   =					os.path.join(__UpperCamelCase							,    F"dataset-{shard_count}-{records_containing}.tfrecord"      )
  UpperCAmelCase__					:  Optional[int]					   =					get_serialized_examples(__UpperCamelCase      )

  with tf.io.TFRecordWriter(__UpperCamelCase      ) as out_file:
   for i in range(len(__UpperCamelCase      )      ):
    UpperCAmelCase__					:  Optional[Any]					   =					serialized_examples[i]
    out_file.write(__UpperCamelCase      )
   print("""Wrote file {} containing {} records""".format(__UpperCamelCase							,    __UpperCamelCase      )      )

  shard_count += 1
  total_records += records_containing

 with open(F"split-{args.split}-records-count.txt"							,    """w"""      ) as f:
  print(F"Total {args.split} records: {total_records}"							,    file=__UpperCamelCase      )


if __name__ == "__main__":
      __UpperCAmelCase =				parse_args()
      main(args)








"""simple docstring"""




# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import sys
import warnings
from os.path import abspath, dirname, join


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
__UpperCAmelCase =				abspath(join(dirname(dirname(__file__)), 'src'))
sys.path.insert(1, git_repo_path)

# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action='ignore', category=FutureWarning)
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 from diffusers.utils.testing_utils import pytest_addoption_shared

 pytest_addoption_shared(__UpperCamelCase      )




def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 from diffusers.utils.testing_utils import pytest_terminal_summary_main

 UpperCAmelCase__					:  Optional[Any]					   =					terminalreporter.config.getoption("""--make-reports"""      )
 if make_reports:
  pytest_terminal_summary_main(__UpperCamelCase							,    id=__UpperCamelCase      )





