




'''simple docstring'''


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCAmelCase_					=   logging.getLogger(__name__)


def     SCREAMING_SNAKE_CASE							(   ):
  __a								=       argparse.ArgumentParser(
      description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'      )
  parser.add_argument(
      '--dataset_name'					,       type=a_					,       default='wikitext'					,       help='Name of the training. Explore datasets at: hf.co/datasets.'					,       )
  parser.add_argument(
      '--dataset_config'					,       type=a_					,       default='wikitext-103-raw-v1'					,       help='Configuration name of the dataset.'      )
  parser.add_argument(
      '--tokenizer_name_or_path'					,       type=a_					,       default='sayakpaul/unigram-tokenizer-wikitext'					,       help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'					,       )
  parser.add_argument(
      '--shard_size'					,       type=a_					,       default=1000					,       help='Number of entries to go in a single shard.'					,       )
  parser.add_argument('--split'					,       type=a_					,       default='train'					,       choices=['train', 'test', 'validation']      )
  parser.add_argument(
      '--limit'					,       default=a_					,       type=a_					,       help='Limit the number of shards (used for debugging).'					,       )
  parser.add_argument(
      '--max_length'					,       type=a_					,       default=512					,       help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
      ' sequence length that is a multiple of 8.'					,       )
  parser.add_argument(
      '--output_dir'					,       default='tf-tpu'					,       type=a_					,       help='Output directory where the TFRecord shards will be saved. If the'
      ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
      ' shards will be directly saved to a Google Cloud Storage bucket.'					,       )

  __a								=       parser.parse_args()
  return args


def     SCREAMING_SNAKE_CASE							(   a_						:				Union[str, Any]      ):
  def fn(a_						:				Dict      ):
    return tokenizer(examples['text']      )

  return fn


def     SCREAMING_SNAKE_CASE							(   a_						:				List[Any]      ):
  __a								=       []
  for i in range(len(tokenized_data['input_ids']      )      ):
    __a								=       {
        'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]      )      ),
        'attention_mask': tf.train.Feature(
            intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]      )      ),
    }
    __a								=       tf.train.Features(feature=a_      )
    __a								=       tf.train.Example(features=a_      )
    __a								=       example.SerializeToString()
    records.append(a_      )
  return records


def     SCREAMING_SNAKE_CASE							(   a_						:				str      ):
  __a								=       datasets.load_dataset(args.dataset_name					,       args.dataset_config					,       split=args.split      )

  if args.limit is not None:
    __a								=       min(len(a_      )					,       args.limit      )
    __a								=       dataset.select(range(a_      )      )
    print(f"Limiting the dataset to {args.limit} entries."      )

  __a								=       AutoTokenizer.from_pretrained(args.tokenizer_name_or_path      )

  # Handle output directory creation.
  # For serializing into a Google Cloud Storage Bucket, one needs to first
  # create a bucket.
  if "gs" not in args.output_dir:
    if not os.path.exists(args.output_dir      ):
      os.makedirs(args.output_dir      )
    __a								=       os.path.join(args.output_dir					,       args.split      )
    if not os.path.exists(a_      ):
      os.makedirs(a_      )
  else:
    __a								=       os.path.join(args.output_dir					,       args.split      )

  # Tokenize the whole dataset at once.
  __a								=       tokenize_function(a_      )
  __a								=       dataset.map(a_					,       batched=a_					,       num_proc=4					,       remove_columns=['text']      )

  # We need to concatenate all our texts together, and then split the result
  # into chunks of a fixed size, which we will call block_size. To do this, we
  # will use the map method again, with the option batched=True. When we use batched=True,
  # the function we pass to map() will be passed multiple inputs at once, allowing us
  # to group them into more or fewer examples than we had in the input.
  # This allows us to create our new fixed-length samples. The advantage of this
  # method is that we don't lose a whole lot of content from the dataset compared to the
  # case where we simply tokenize with a pre-defined max_length.

  def group_texts(a_						:				Optional[Any]      ):
    # Concatenate all texts.
    __a								=       {k: sum(examples[k]					,       []      ) for k in examples.keys()}
    __a								=       len(concatenated_examples[list(examples.keys()      )[0]]      )
    # We drop the small remainder, though you could add padding instead if the model supports it
    # In this, as in all things, we advise you to follow your heart 🫀
    __a								=       (total_length // args.max_length) * args.max_length
    # Split by chunks of max_len.
    __a								=       {
        k: [t[i : i + args.max_length] for i in range(0					,       a_					,       args.max_length      )]
        for k, t in concatenated_examples.items()
    }
    return result

  __a								=       dataset_tokenized.map(a_					,       batched=a_					,       batch_size=1000					,       num_proc=4      )

  __a								=       0
  __a								=       0
  for shard in range(0					,       len(a_      )					,       args.shard_size      ):
    __a								=       grouped_dataset[shard : shard + args.shard_size]
    __a								=       len(dataset_snapshot['input_ids']      )
    __a								=       os.path.join(a_					,       f"dataset-{shard_count}-{records_containing}.tfrecord"      )
    __a								=       get_serialized_examples(a_      )

    with tf.io.TFRecordWriter(a_      ) as out_file:
      for i in range(len(a_      )      ):
        __a								=       serialized_examples[i]
        out_file.write(a_      )
      print('Wrote file {} containing {} records'.format(a_					,       a_      )      )

    shard_count += 1
    total_records += records_containing

  with open(f"split-{args.split}-records-count.txt"					,       'w'      ) as f:
    print(f"Total {args.split} records: {total_records}"					,       file=a_      )


if __name__ == "__main__":
  UpperCAmelCase_					=   parse_args()
  main(args)









'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


UpperCAmelCase_					=   {"configuration_swin": ["SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP", "SwinConfig", "SwinOnnxConfig"]}


try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase_					=   [
      "SWIN_PRETRAINED_MODEL_ARCHIVE_LIST",
      "SwinForImageClassification",
      "SwinForMaskedImageModeling",
      "SwinModel",
      "SwinPreTrainedModel",
      "SwinBackbone",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase_					=   [
      "TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST",
      "TFSwinForImageClassification",
      "TFSwinForMaskedImageModeling",
      "TFSwinModel",
      "TFSwinPreTrainedModel",
  ]

if TYPE_CHECKING:
  from .configuration_swin import SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, SwinConfig, SwinOnnxConfig

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_swin import (
        SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
        SwinBackbone,
        SwinForImageClassification,
        SwinForMaskedImageModeling,
        SwinModel,
        SwinPreTrainedModel,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_swin import (
        TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFSwinForImageClassification,
        TFSwinForMaskedImageModeling,
        TFSwinModel,
        TFSwinPreTrainedModel,
    )

else:
  import sys

  UpperCAmelCase_					=   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



