






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


snake_case     					=		logging.getLogger(__name__)


def 	lowerCamelCase__      (  ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE							:   List[str]						    =     argparse.ArgumentParser(
      description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."					)
  parser.add_argument(
      "--dataset_name"    ,  type=lowercase    ,  default="wikitext"    ,  help="Name of the training. Explore datasets at: hf.co/datasets."    ,  )
  parser.add_argument(
      "--dataset_config"    ,  type=lowercase    ,  default="wikitext-103-raw-v1"    ,  help="Configuration name of the dataset."					)
  parser.add_argument(
      "--tokenizer_name_or_path"    ,  type=lowercase    ,  default="sayakpaul/unigram-tokenizer-wikitext"    ,  help="Tokenizer identifier. Can be a local filepath or a Hub identifier."    ,  )
  parser.add_argument(
      "--shard_size"    ,  type=lowercase    ,  default=1000    ,  help="Number of entries to go in a single shard."    ,  )
  parser.add_argument("--split"    ,  type=lowercase    ,  default="train"    ,  choices=["train", "test", "validation"]					)
  parser.add_argument(
      "--limit"    ,  default=lowercase    ,  type=lowercase    ,  help="Limit the number of shards (used for debugging)."    ,  )
  parser.add_argument(
      "--max_length"    ,  type=lowercase    ,  default=512    ,  help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
      " sequence length that is a multiple of 8."    ,  )
  parser.add_argument(
      "--output_dir"    ,  default="tf-tpu"    ,  type=lowercase    ,  help="Output directory where the TFRecord shards will be saved. If the"
      " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
      " shards will be directly saved to a Google Cloud Storage bucket."    ,  )

  SCREAMING_SNAKE_CASE							:   Optional[int]						    =     parser.parse_args()
  return args


def 	lowerCamelCase__      (  lowercase					):

  """simple docstring"""






  def fn(lowercase					):
    return tokenizer(examples["text"]					)

  return fn


def 	lowerCamelCase__      (  lowercase					):

  """simple docstring"""






  SCREAMING_SNAKE_CASE							:   int						    =     []
  for i in range(len(tokenized_data["input_ids"]					)					):
    SCREAMING_SNAKE_CASE							:   Any						    =     {
        "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]					)					),
        "attention_mask": tf.train.Feature(
            intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]					)					),
    }
    SCREAMING_SNAKE_CASE							:   str						    =     tf.train.Features(feature=lowercase					)
    SCREAMING_SNAKE_CASE							:   List[Any]						    =     tf.train.Example(features=lowercase					)
    SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     example.SerializeToString()
    records.append(lowercase					)
  return records


def 	lowerCamelCase__      (  lowercase					):

  """simple docstring"""






  SCREAMING_SNAKE_CASE							:   Tuple						    =     datasets.load_dataset(args.dataset_name    ,  args.dataset_config    ,  split=args.split					)

  if args.limit is not None:
    SCREAMING_SNAKE_CASE							:   Union[str, Any]						    =     min(len(lowercase					)    ,  args.limit					)
    SCREAMING_SNAKE_CASE							:   List[str]						    =     dataset.select(range(lowercase					)					)
    print(F'''Limiting the dataset to {args.limit} entries.'''					)

  SCREAMING_SNAKE_CASE							:   Any						    =     AutoTokenizer.from_pretrained(args.tokenizer_name_or_path					)

  # Handle output directory creation.
  # For serializing into a Google Cloud Storage Bucket, one needs to first
  # create a bucket.
  if "gs" not in args.output_dir:
    if not os.path.exists(args.output_dir					):
      os.makedirs(args.output_dir					)
    SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     os.path.join(args.output_dir    ,  args.split					)
    if not os.path.exists(lowercase					):
      os.makedirs(lowercase					)
  else:
    SCREAMING_SNAKE_CASE							:   List[str]						    =     os.path.join(args.output_dir    ,  args.split					)

  # Tokenize the whole dataset at once.
  SCREAMING_SNAKE_CASE							:   Union[str, Any]						    =     tokenize_function(lowercase					)
  SCREAMING_SNAKE_CASE							:   List[Any]						    =     dataset.map(lowercase    ,  batched=lowercase    ,  num_proc=4    ,  remove_columns=["text"]					)

  # We need to concatenate all our texts together, and then split the result
  # into chunks of a fixed size, which we will call block_size. To do this, we
  # will use the map method again, with the option batched=True. When we use batched=True,
  # the function we pass to map() will be passed multiple inputs at once, allowing us
  # to group them into more or fewer examples than we had in the input.
  # This allows us to create our new fixed-length samples. The advantage of this
  # method is that we don't lose a whole lot of content from the dataset compared to the
  # case where we simply tokenize with a pre-defined max_length.

  def group_texts(lowercase					):
    # Concatenate all texts.
    SCREAMING_SNAKE_CASE							:   Union[str, Any]						    =     {k: sum(examples[k]    ,  []					) for k in examples.keys()}
    SCREAMING_SNAKE_CASE							:   List[Any]						    =     len(concatenated_examples[list(examples.keys()					)[0]]					)
    # We drop the small remainder, though you could add padding instead if the model supports it
    # In this, as in all things, we advise you to follow your heart 🫀
    SCREAMING_SNAKE_CASE							:   List[Any]						    =     (total_length // args.max_length) * args.max_length
    # Split by chunks of max_len.
    SCREAMING_SNAKE_CASE							:   List[Any]						    =     {
        k: [t[i : i + args.max_length] for i in range(0    ,  lowercase    ,  args.max_length					)]
        for k, t in concatenated_examples.items()
    }
    return result

  SCREAMING_SNAKE_CASE							:   Tuple						    =     dataset_tokenized.map(lowercase    ,  batched=lowercase    ,  batch_size=1000    ,  num_proc=4					)

  SCREAMING_SNAKE_CASE							:   str						    =     0
  SCREAMING_SNAKE_CASE							:   Optional[int]						    =     0
  for shard in range(0    ,  len(lowercase					)    ,  args.shard_size					):
    SCREAMING_SNAKE_CASE							:   int						    =     grouped_dataset[shard : shard + args.shard_size]
    SCREAMING_SNAKE_CASE							:   Dict						    =     len(dataset_snapshot["input_ids"]					)
    SCREAMING_SNAKE_CASE							:   Any						    =     os.path.join(lowercase    ,  F'''dataset-{shard_count}-{records_containing}.tfrecord'''					)
    SCREAMING_SNAKE_CASE							:   Any						    =     get_serialized_examples(lowercase					)

    with tf.io.TFRecordWriter(lowercase					) as out_file:
      for i in range(len(lowercase					)					):
        SCREAMING_SNAKE_CASE							:   List[Any]						    =     serialized_examples[i]
        out_file.write(lowercase					)
      print("Wrote file {} containing {} records".format(lowercase    ,  lowercase					)					)

    shard_count += 1
    total_records += records_containing

  with open(F'''split-{args.split}-records-count.txt'''    ,  "w"					) as f:
    print(F'''Total {args.split} records: {total_records}'''    ,  file=lowercase					)


if __name__ == "__main__":
   snake_case     					=		parse_args()
   main(args)












'''simple docstring'''
from collections import UserDict
from typing import List, Union

from ..utils import (
    add_end_docstrings,
    is_tf_available,
    is_torch_available,
    is_vision_available,
    logging,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_vision_available():
							from PIL import Image

							from ..image_utils import load_image

if is_torch_available():
							from ..models.auto.modeling_auto import MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING

if is_tf_available():
							from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING
							from ..tf_utils import stable_softmax

_SCREAMING_SNAKE_CASE			=						logging.get_logger(__name__)
@add_end_docstrings(lowerCAmelCase				)
class   __UpperCAmelCase				(     lowerCAmelCase				):



					'''simple docstring'''





					def __init__(	self     :						int      ,       **_lowercase     :						List[Any])    ->							Union[str, Any]:
								super().__init__(**_lowercase)

								requires_backends(self      ,       'vision')
								self.check_model_type(
								    TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING
								    if self.framework == 'tf'
								    else MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING)



					def __call__(	self     :						Optional[Any]      ,       _lowercase     :						Union[str, List[str], "Image", List["Image"]]      ,       **_lowercase     :						List[Any])    ->							Any:
								return super().__call__(_lowercase      ,       **_lowercase)



					def     __snake_case						(	self     :						int      ,       **_lowercase     :						Union[str, Any])    ->							Any:
								A_									= {}
								if "candidate_labels" in kwargs:
											A_									= kwargs['candidate_labels']
								if "hypothesis_template" in kwargs:
											A_									= kwargs['hypothesis_template']

								return preprocess_params, {}, {}



					def     __snake_case						(	self     :						List[str]      ,       _lowercase     :						Tuple      ,       _lowercase     :						Any=None      ,       _lowercase     :						Optional[int]="This is a photo of {}.")    ->							Union[str, Any]:
								A_									= load_image(_lowercase)
								A_									= self.image_processor(images=[image]      ,       return_tensors=self.framework)
								A_									= candidate_labels
								A_									= [hypothesis_template.format(_lowercase) for x in candidate_labels]
								A_									= self.tokenizer(_lowercase      ,       return_tensors=self.framework      ,       padding=_lowercase)
								A_									= [text_inputs]
								return inputs



					def     __snake_case						(	self     :						Optional[int]      ,       _lowercase     :						Tuple)    ->							Optional[int]:
								A_									= model_inputs.pop('candidate_labels')
								A_									= model_inputs.pop('text_inputs')
								if isinstance(text_inputs[0]      ,       _lowercase):
											A_									= text_inputs[0]
								else:
											# Batching case.
											A_									= text_inputs[0][0]

								A_									= self.model(**_lowercase      ,       **_lowercase)

								A_									= {
								    'candidate_labels': candidate_labels,
								    'logits': outputs.logits_per_image,
								}
								return model_outputs



					def     __snake_case						(	self     :						List[str]      ,       _lowercase     :						int)    ->							Optional[int]:
								A_									= model_outputs.pop('candidate_labels')
								A_									= model_outputs['logits'][0]
								if self.framework == "pt":
											A_									= logits.softmax(dim=-1).squeeze(-1)
											A_									= probs.tolist()
											if not isinstance(_lowercase      ,       _lowercase):
														A_									= [scores]
								elif self.framework == "tf":
											A_									= stable_softmax(_lowercase      ,       axis=-1)
											A_									= probs.numpy().tolist()
								else:
											raise ValueError(F'Unsupported framework: {self.framework}')

								A_									= [
								    {'score': score, 'label': candidate_label}
								    for score, candidate_label in sorted(zip(_lowercase      ,       _lowercase)      ,       key=lambda _lowercase: -x[0])
								]
								return result




