
"""simple docstring"""



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCAmelCase__        =					logging.getLogger(__name__)
def    _UpperCAmelCase (					)	->   Optional[Any]:
  _snake_case   	=			argparse.ArgumentParser(
      description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''						)
  parser.add_argument(
      '''--dataset_name'''					,   type=__lowerCamelCase					,   default='''wikitext'''					,   help='''Name of the training. Explore datasets at: hf.co/datasets.'''					,   )
  parser.add_argument(
      '''--dataset_config'''					,   type=__lowerCamelCase					,   default='''wikitext-103-raw-v1'''					,   help='''Configuration name of the dataset.'''						)
  parser.add_argument(
      '''--tokenizer_name_or_path'''					,   type=__lowerCamelCase					,   default='''sayakpaul/unigram-tokenizer-wikitext'''					,   help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''					,   )
  parser.add_argument(
      '''--shard_size'''					,   type=__lowerCamelCase					,   default=10_00					,   help='''Number of entries to go in a single shard.'''					,   )
  parser.add_argument('''--split'''					,   type=__lowerCamelCase					,   default='''train'''					,   choices=['''train''', '''test''', '''validation''']						)
  parser.add_argument(
      '''--limit'''					,   default=__lowerCamelCase					,   type=__lowerCamelCase					,   help='''Limit the number of shards (used for debugging).'''					,   )
  parser.add_argument(
      '''--max_length'''					,   type=__lowerCamelCase					,   default=5_12					,   help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
      ''' sequence length that is a multiple of 8.'''					,   )
  parser.add_argument(
      '''--output_dir'''					,   default='''tf-tpu'''					,   type=__lowerCamelCase					,   help='''Output directory where the TFRecord shards will be saved. If the'''
      ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
      ''' shards will be directly saved to a Google Cloud Storage bucket.'''					,   )

  _snake_case   	=			parser.parse_args()
  return args
def    _UpperCAmelCase (					__lowerCamelCase  :			Optional[int]						)	->   Optional[Any]:
  def fn(__lowerCamelCase  :			List[Any]						):
    return tokenizer(examples['''text''']						)

  return fn
def    _UpperCAmelCase (					__lowerCamelCase  :			List[str]						)	->   Optional[Any]:
  _snake_case   	=			[]
  for i in range(len(tokenized_data['''input_ids''']						)						):
    _snake_case   	=			{
        '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]						)						),
        '''attention_mask''': tf.train.Feature(
            intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]						)						),
    }
    _snake_case   	=			tf.train.Features(feature=__lowerCamelCase						)
    _snake_case   	=			tf.train.Example(features=__lowerCamelCase						)
    _snake_case   	=			example.SerializeToString()
    records.append(__lowerCamelCase						)
  return records
def    _UpperCAmelCase (					__lowerCamelCase  :			List[str]						)	->   Tuple:
  _snake_case   	=			datasets.load_dataset(args.dataset_name					,   args.dataset_config					,   split=args.split						)

  if args.limit is not None:
    _snake_case   	=			min(len(__lowerCamelCase						)					,   args.limit						)
    _snake_case   	=			dataset.select(range(__lowerCamelCase						)						)
    print(f'''Limiting the dataset to {args.limit} entries.'''						)

  _snake_case   	=			AutoTokenizer.from_pretrained(args.tokenizer_name_or_path						)

  # Handle output directory creation.
  # For serializing into a Google Cloud Storage Bucket, one needs to first
  # create a bucket.
  if "gs" not in args.output_dir:
    if not os.path.exists(args.output_dir						):
      os.makedirs(args.output_dir						)
    _snake_case   	=			os.path.join(args.output_dir					,   args.split						)
    if not os.path.exists(__lowerCamelCase						):
      os.makedirs(__lowerCamelCase						)
  else:
    _snake_case   	=			os.path.join(args.output_dir					,   args.split						)

  # Tokenize the whole dataset at once.
  _snake_case   	=			tokenize_function(__lowerCamelCase						)
  _snake_case   	=			dataset.map(__lowerCamelCase					,   batched=__lowerCamelCase					,   num_proc=4					,   remove_columns=['''text''']						)

  # We need to concatenate all our texts together, and then split the result
  # into chunks of a fixed size, which we will call block_size. To do this, we
  # will use the map method again, with the option batched=True. When we use batched=True,
  # the function we pass to map() will be passed multiple inputs at once, allowing us
  # to group them into more or fewer examples than we had in the input.
  # This allows us to create our new fixed-length samples. The advantage of this
  # method is that we don't lose a whole lot of content from the dataset compared to the
  # case where we simply tokenize with a pre-defined max_length.

  def group_texts(__lowerCamelCase  :			int						):
    # Concatenate all texts.
    _snake_case   	=			{k: sum(examples[k]					,   []						) for k in examples.keys()}
    _snake_case   	=			len(concatenated_examples[list(examples.keys()						)[0]]						)
    # We drop the small remainder, though you could add padding instead if the model supports it
    # In this, as in all things, we advise you to follow your heart 🫀
    _snake_case   	=			(total_length // args.max_length) * args.max_length
    # Split by chunks of max_len.
    _snake_case   	=			{
        k: [t[i : i + args.max_length] for i in range(0					,   __lowerCamelCase					,   args.max_length						)]
        for k, t in concatenated_examples.items()
    }
    return result

  _snake_case   	=			dataset_tokenized.map(__lowerCamelCase					,   batched=__lowerCamelCase					,   batch_size=10_00					,   num_proc=4						)

  _snake_case   	=			0
  _snake_case   	=			0
  for shard in range(0					,   len(__lowerCamelCase						)					,   args.shard_size						):
    _snake_case   	=			grouped_dataset[shard : shard + args.shard_size]
    _snake_case   	=			len(dataset_snapshot['''input_ids''']						)
    _snake_case   	=			os.path.join(__lowerCamelCase					,   f'''dataset-{shard_count}-{records_containing}.tfrecord'''						)
    _snake_case   	=			get_serialized_examples(__lowerCamelCase						)

    with tf.io.TFRecordWriter(__lowerCamelCase						) as out_file:
      for i in range(len(__lowerCamelCase						)						):
        _snake_case   	=			serialized_examples[i]
        out_file.write(__lowerCamelCase						)
      print('''Wrote file {} containing {} records'''.format(__lowerCamelCase					,   __lowerCamelCase						)						)

    shard_count += 1
    total_records += records_containing

  with open(f'''split-{args.split}-records-count.txt'''					,   '''w'''						) as f:
    print(f'''Total {args.split} records: {total_records}'''					,   file=__lowerCamelCase						)


if __name__ == "__main__":
 UpperCAmelCase__        =					parse_args()
 main(args)




from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import rescale, resize, to_channel_dimension_format
from ...image_utils import (
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
  import PIL


UpperCAmelCase_           =				logging.get_logger(__name__)




def    lowerCAmelCase_  (		lowercase:				str	,       lowercase:				Union[str, Any]		)     ->					Optional[int]:







 '''simple docstring'''
 _UpperCamelCase:  int    						=			b.T
 _UpperCamelCase:  Optional[Any]    						=			np.sum(np.square(lowercase		)	,       axis=1		)
 _UpperCamelCase:  List[Any]    						=			np.sum(np.square(lowercase		)	,       axis=0		)
 _UpperCamelCase:  Tuple    						=			np.matmul(lowercase	,       lowercase		)
 _UpperCamelCase:  Tuple    						=			aa[:, None] - 2 * ab + ba[None, :]
 return d




def    lowerCAmelCase_  (		lowercase:				List[str]	,       lowercase:				List[Any]		)     ->					List[Any]:







 '''simple docstring'''
 _UpperCamelCase:  Tuple    						=			x.reshape(-1	,       3		)
 _UpperCamelCase:  Dict    						=			squared_euclidean_distance(lowercase	,       lowercase		)
 return np.argmin(lowercase	,       axis=1		)





class 			__magic_name__	(		__a     ):





   """simple docstring"""




   lowerCAmelCase   :     Optional[Any]	      = ['''pixel_values''']



   def __init__(						self       :	List[str]		,	_lowercase       :	Optional[Union[List[List[int]], np.ndarray]] = None		,	_lowercase       :	bool = True		,	_lowercase       :	Dict[str, int] = None		,	_lowercase       :	PILImageResampling = PILImageResampling.BILINEAR		,	_lowercase       :	bool = True		,	_lowercase       :	bool = True		,	**_lowercase       :	Tuple		,	):



    """simple docstring"""



    super().__init__(**_lowercase				)
    _UpperCamelCase:  Tuple    						=			size if size is not None else {'''height''': 256, '''width''': 256}
    _UpperCamelCase:  List[str]    						=			get_size_dict(_lowercase				)
    _UpperCamelCase:  Dict    						=			np.array(_lowercase				) if clusters is not None else None
    _UpperCamelCase:  Optional[Any]    						=			do_resize
    _UpperCamelCase:  Union[str, Any]    						=			size
    _UpperCamelCase:  Optional[int]    						=			resample
    _UpperCamelCase:  Any    						=			do_normalize
    _UpperCamelCase:  int    						=			do_color_quantize



   def 	lowerCAmelCase    (						self       :	List[Any]		,	_lowercase       :	np.ndarray		,	_lowercase       :	Dict[str, int]		,	_lowercase       :	PILImageResampling = PILImageResampling.BILINEAR		,	_lowercase       :	Optional[Union[str, ChannelDimension]] = None		,	**_lowercase       :	Tuple		,	):



    """simple docstring"""



    _UpperCamelCase:  List[str]    						=			get_size_dict(_lowercase				)
    if "height" not in size or "width" not in size:
     raise ValueError(f"""Size dictionary must contain both height and width keys. Got {size.keys()}"""				)
    return resize(
        _lowercase		,	size=(size['''height'''], size['''width'''])		,	resample=_lowercase		,	data_format=_lowercase		,	**_lowercase				)



   def 	lowerCAmelCase    (						self       :	Optional[int]		,	_lowercase       :	np.ndarray		,	_lowercase       :	Optional[Union[str, ChannelDimension]] = None		,	):



    """simple docstring"""



    _UpperCamelCase:  Any    						=			rescale(image=_lowercase		,	scale=1 / 127.5		,	data_format=_lowercase				)
    _UpperCamelCase:  List[Any]    						=			image - 1
    return image



   def 	lowerCAmelCase    (						self       :	Optional[int]		,	_lowercase       :	ImageInput		,	_lowercase       :	bool = None		,	_lowercase       :	Dict[str, int] = None		,	_lowercase       :	PILImageResampling = None		,	_lowercase       :	bool = None		,	_lowercase       :	Optional[bool] = None		,	_lowercase       :	Optional[Union[List[List[int]], np.ndarray]] = None		,	_lowercase       :	Optional[Union[str, TensorType]] = None		,	_lowercase       :	Optional[Union[str, ChannelDimension]] = ChannelDimension.FIRST		,	**_lowercase       :	List[str]		,	):



    """simple docstring"""



    _UpperCamelCase:  Any    						=			do_resize if do_resize is not None else self.do_resize
    _UpperCamelCase:  Optional[Any]    						=			size if size is not None else self.size
    _UpperCamelCase:  Optional[Any]    						=			get_size_dict(_lowercase				)
    _UpperCamelCase:  Optional[int]    						=			resample if resample is not None else self.resample
    _UpperCamelCase:  Union[str, Any]    						=			do_normalize if do_normalize is not None else self.do_normalize
    _UpperCamelCase:  Any    						=			do_color_quantize if do_color_quantize is not None else self.do_color_quantize
    _UpperCamelCase:  Dict    						=			clusters if clusters is not None else self.clusters
    _UpperCamelCase:  Optional[Any]    						=			np.array(_lowercase				)

    _UpperCamelCase:  List[str]    						=			make_list_of_images(_lowercase				)

    if not valid_images(_lowercase				):
     raise ValueError(
         '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
         '''torch.Tensor, tf.Tensor or jax.ndarray.'''				)

    if do_resize and size is None or resample is None:
     raise ValueError('''Size and resample must be specified if do_resize is True.'''				)

    if do_color_quantize and clusters is None:
     raise ValueError('''Clusters must be specified if do_color_quantize is True.'''				)

    # All transformations expect numpy arrays.
    _UpperCamelCase:  Optional[int]    						=			[to_numpy_array(_lowercase				) for image in images]

    if do_resize:
     _UpperCamelCase:  int    						=			[self.resize(image=_lowercase		,	size=_lowercase		,	resample=_lowercase				) for image in images]

    if do_normalize:
     _UpperCamelCase:  Any    						=			[self.normalize(image=_lowercase				) for image in images]

    if do_color_quantize:
     _UpperCamelCase:  Tuple    						=			[to_channel_dimension_format(_lowercase		,	ChannelDimension.LAST				) for image in images]
     # color quantize from (batch_size, height, width, 3) to (batch_size, height, width)
     _UpperCamelCase:  Tuple    						=			np.array(_lowercase				)
     _UpperCamelCase:  List[Any]    						=			color_quantize(_lowercase		,	_lowercase				).reshape(images.shape[:-1]				)

     # flatten to (batch_size, height*width)
     _UpperCamelCase:  Any    						=			images.shape[0]
     _UpperCamelCase:  Optional[int]    						=			images.reshape(_lowercase		,	-1				)

     # We need to convert back to a list of images to keep consistent behaviour across processors.
     _UpperCamelCase:  Tuple    						=			list(_lowercase				)
    else:
     _UpperCamelCase:  List[str]    						=			[to_channel_dimension_format(_lowercase		,	_lowercase				) for image in images]

    _UpperCamelCase:  List[Any]    						=			{'''input_ids''': images}
    return BatchFeature(data=_lowercase		,	tensor_type=_lowercase				)