





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCAmelCase_    =    logging.getLogger(__name__)






def 					lowerCamelCase_						(    ):






				"""simple docstring"""


				snake_case_						:   List[Any]           =  argparse.ArgumentParser(
				    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''			)
				parser.add_argument(
				    '''--dataset_name'''			,    type=lowerCamelCase_			,    default='''wikitext'''			,    help='''Name of the training. Explore datasets at: hf.co/datasets.'''			,    )
				parser.add_argument(
				    '''--dataset_config'''			,    type=lowerCamelCase_			,    default='''wikitext-103-raw-v1'''			,    help='''Configuration name of the dataset.'''			)
				parser.add_argument(
				    '''--tokenizer_name_or_path'''			,    type=lowerCamelCase_			,    default='''sayakpaul/unigram-tokenizer-wikitext'''			,    help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''			,    )
				parser.add_argument(
				    '''--shard_size'''			,    type=lowerCamelCase_			,    default=1_000			,    help='''Number of entries to go in a single shard.'''			,    )
				parser.add_argument('''--split'''			,    type=lowerCamelCase_			,    default='''train'''			,    choices=['''train''', '''test''', '''validation''']			)
				parser.add_argument(
				    '''--limit'''			,    default=lowerCamelCase_			,    type=lowerCamelCase_			,    help='''Limit the number of shards (used for debugging).'''			,    )
				parser.add_argument(
				    '''--max_length'''			,    type=lowerCamelCase_			,    default=512			,    help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
				    ''' sequence length that is a multiple of 8.'''			,    )
				parser.add_argument(
				    '''--output_dir'''			,    default='''tf-tpu'''			,    type=lowerCamelCase_			,    help='''Output directory where the TFRecord shards will be saved. If the'''
				    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
				    ''' shards will be directly saved to a Google Cloud Storage bucket.'''			,    )

				snake_case_						:   Optional[int]           =  parser.parse_args()
				return args






def 					lowerCamelCase_						(    _UpperCamelCase			):






				"""simple docstring"""


				def fn(_UpperCamelCase			):
								return tokenizer(examples['''text''']			)

				return fn






def 					lowerCamelCase_						(    _UpperCamelCase			):






				"""simple docstring"""


				snake_case_						:   Optional[Any]           =  []
				for i in range(len(tokenized_data['''input_ids''']			)			):
								snake_case_						:   List[Any]           =  {
								    """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]			)			),
								    """attention_mask""": tf.train.Feature(
								        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]			)			),
								}
								snake_case_						:   Tuple           =  tf.train.Features(feature=lowerCamelCase_			)
								snake_case_						:   Union[str, Any]           =  tf.train.Example(features=lowerCamelCase_			)
								snake_case_						:   List[Any]           =  example.SerializeToString()
								records.append(lowerCamelCase_			)
				return records






def 					lowerCamelCase_						(    _UpperCamelCase			):






				"""simple docstring"""


				snake_case_						:   List[str]           =  datasets.load_dataset(args.dataset_name			,    args.dataset_config			,    split=args.split			)

				if args.limit is not None:
								snake_case_						:   List[str]           =  min(len(lowerCamelCase_			)			,    args.limit			)
								snake_case_						:   Optional[Any]           =  dataset.select(range(lowerCamelCase_			)			)
								print(f'''Limiting the dataset to {args.limit} entries.'''			)

				snake_case_						:   Tuple           =  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path			)

				# Handle output directory creation.
				# For serializing into a Google Cloud Storage Bucket, one needs to first
				# create a bucket.
				if "gs" not in args.output_dir:
								if not os.path.exists(args.output_dir			):
												os.makedirs(args.output_dir			)
								snake_case_						:   int           =  os.path.join(args.output_dir			,    args.split			)
								if not os.path.exists(lowerCamelCase_			):
												os.makedirs(lowerCamelCase_			)
				else:
								snake_case_						:   List[str]           =  os.path.join(args.output_dir			,    args.split			)

				# Tokenize the whole dataset at once.
				snake_case_						:   List[str]           =  tokenize_function(lowerCamelCase_			)
				snake_case_						:   Optional[int]           =  dataset.map(lowerCamelCase_			,    batched=lowerCamelCase_			,    num_proc=4			,    remove_columns=['''text''']			)

				# We need to concatenate all our texts together, and then split the result
				# into chunks of a fixed size, which we will call block_size. To do this, we
				# will use the map method again, with the option batched=True. When we use batched=True,
				# the function we pass to map() will be passed multiple inputs at once, allowing us
				# to group them into more or fewer examples than we had in the input.
				# This allows us to create our new fixed-length samples. The advantage of this
				# method is that we don't lose a whole lot of content from the dataset compared to the
				# case where we simply tokenize with a pre-defined max_length.

				def group_texts(_UpperCamelCase			):
								# Concatenate all texts.
								snake_case_						:   Any           =  {k: sum(examples[k]			,    []			) for k in examples.keys()}
								snake_case_						:   Tuple           =  len(concatenated_examples[list(examples.keys()			)[0]]			)
								# We drop the small remainder, though you could add padding instead if the model supports it
								# In this, as in all things, we advise you to follow your heart 🫀
								snake_case_						:   str           =  (total_length // args.max_length) * args.max_length
								# Split by chunks of max_len.
								snake_case_						:   Tuple           =  {
								    k: [t[i : i + args.max_length] for i in range(0			,    lowerCamelCase_			,    args.max_length			)]
								    for k, t in concatenated_examples.items()
								}
								return result

				snake_case_						:   List[str]           =  dataset_tokenized.map(lowerCamelCase_			,    batched=lowerCamelCase_			,    batch_size=1_000			,    num_proc=4			)

				snake_case_						:   Optional[Any]           =  0
				snake_case_						:   str           =  0
				for shard in range(0			,    len(lowerCamelCase_			)			,    args.shard_size			):
								snake_case_						:   List[Any]           =  grouped_dataset[shard : shard + args.shard_size]
								snake_case_						:   Dict           =  len(dataset_snapshot['''input_ids''']			)
								snake_case_						:   int           =  os.path.join(lowerCamelCase_			,    f'''dataset-{shard_count}-{records_containing}.tfrecord'''			)
								snake_case_						:   List[Any]           =  get_serialized_examples(lowerCamelCase_			)

								with tf.io.TFRecordWriter(lowerCamelCase_			) as out_file:
												for i in range(len(lowerCamelCase_			)			):
																snake_case_						:   Dict           =  serialized_examples[i]
																out_file.write(lowerCamelCase_			)
												print('''Wrote file {} containing {} records'''.format(lowerCamelCase_			,    lowerCamelCase_			)			)

								shard_count += 1
								total_records += records_containing

				with open(f'''split-{args.split}-records-count.txt'''			,    '''w'''			) as f:
								print(f'''Total {args.split} records: {total_records}'''			,    file=lowerCamelCase_			)


if __name__ == "__main__":
	lowerCAmelCase_    =    parse_args()
	main(args)







import os
from typing import List, Optional, Union

from ...image_processing_utils import BatchFeature
from ...image_utils import ImageInput
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType
from ..auto import AutoTokenizer



class  __lowerCAmelCase	(      _a					):
			lowerCamelCase_					:					Optional[Any]	      =				['''image_processor''', '''tokenizer''']
			lowerCamelCase_					:					List[Any]	      =				'''BlipImageProcessor'''
			lowerCamelCase_					:					Union[str, Any]	      =				'''AutoTokenizer'''






			def __init__(self ,							__magic_name__ ,							__magic_name__ ,							__magic_name__   )					->       Dict:


							'''simple docstring'''



							super().__init__(__magic_name__ ,							__magic_name__   )

							# add QFormer tokenizer
							snake_case_						:   Optional[Any]           =  qformer_tokenizer






			def __call__(self ,							__magic_name__ = None ,							__magic_name__ = None ,							__magic_name__ = True ,							__magic_name__ = False ,							__magic_name__ = None ,							__magic_name__ = None ,							__magic_name__ = 0 ,							__magic_name__ = None ,							__magic_name__ = None ,							__magic_name__ = False ,							__magic_name__ = False ,							__magic_name__ = False ,							__magic_name__ = False ,							__magic_name__ = False ,							__magic_name__ = True ,							__magic_name__ = None ,							**__magic_name__ ,							)					->       BatchFeature:


							'''simple docstring'''



							if images is None and text is None:
											raise ValueError('''You have to specify at least images or text.'''   )

							snake_case_						:   Tuple           =  BatchFeature()

							if text is not None:
											snake_case_						:   Tuple           =  self.tokenizer(
											    text=__magic_name__ ,							add_special_tokens=__magic_name__ ,							padding=__magic_name__ ,							truncation=__magic_name__ ,							max_length=__magic_name__ ,							stride=__magic_name__ ,							pad_to_multiple_of=__magic_name__ ,							return_attention_mask=__magic_name__ ,							return_overflowing_tokens=__magic_name__ ,							return_special_tokens_mask=__magic_name__ ,							return_offsets_mapping=__magic_name__ ,							return_token_type_ids=__magic_name__ ,							return_length=__magic_name__ ,							verbose=__magic_name__ ,							return_tensors=__magic_name__ ,							**__magic_name__ ,							)
											encoding.update(__magic_name__   )
											snake_case_						:   Optional[Any]           =  self.qformer_tokenizer(
											    text=__magic_name__ ,							add_special_tokens=__magic_name__ ,							padding=__magic_name__ ,							truncation=__magic_name__ ,							max_length=__magic_name__ ,							stride=__magic_name__ ,							pad_to_multiple_of=__magic_name__ ,							return_attention_mask=__magic_name__ ,							return_overflowing_tokens=__magic_name__ ,							return_special_tokens_mask=__magic_name__ ,							return_offsets_mapping=__magic_name__ ,							return_token_type_ids=__magic_name__ ,							return_length=__magic_name__ ,							verbose=__magic_name__ ,							return_tensors=__magic_name__ ,							**__magic_name__ ,							)
											snake_case_						:   Optional[int]           =  qformer_text_encoding.pop('''input_ids'''   )
											snake_case_						:   Tuple           =  qformer_text_encoding.pop('''attention_mask'''   )

							if images is not None:
											snake_case_						:   Any           =  self.image_processor(__magic_name__ ,							return_tensors=__magic_name__   )
											encoding.update(__magic_name__   )

							return encoding






			def       lowerCamelCase  (self ,							*__magic_name__ ,							**__magic_name__   )					->       List[Any]:


							'''simple docstring'''



							return self.tokenizer.batch_decode(*__magic_name__ ,							**__magic_name__   )






			def       lowerCamelCase  (self ,							*__magic_name__ ,							**__magic_name__   )					->       Union[str, Any]:


							'''simple docstring'''



							return self.tokenizer.decode(*__magic_name__ ,							**__magic_name__   )






			@property
			# Copied from transformers.models.blip.processing_blip.BlipProcessor.model_input_names
			def       lowerCamelCase  (self   )					->       Any:


							'''simple docstring'''



							snake_case_						:   Optional[Any]           =  self.tokenizer.model_input_names
							snake_case_						:   List[Any]           =  self.image_processor.model_input_names
							return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names   )   )






			def       lowerCamelCase  (self ,							__magic_name__ ,							**__magic_name__   )					->       List[Any]:


							'''simple docstring'''



							if os.path.isfile(__magic_name__   ):
											raise ValueError(F'''Provided path ({save_directory}) should be a directory, not a file'''   )
							os.makedirs(__magic_name__ ,							exist_ok=__magic_name__   )
							snake_case_						:   Any           =  os.path.join(__magic_name__ ,							'''qformer_tokenizer'''   )
							self.qformer_tokenizer.save_pretrained(__magic_name__   )
							return super().save_pretrained(__magic_name__ ,							**__magic_name__   )






			@classmethod
			def       lowerCamelCase  (cls ,							__magic_name__ ,							**__magic_name__   )					->       Union[str, Any]:


							'''simple docstring'''



							snake_case_						:   Any           =  AutoTokenizer.from_pretrained(__magic_name__ ,							subfolder='''qformer_tokenizer'''   )
							snake_case_						:   str           =  cls._get_arguments_from_pretrained(__magic_name__ ,							**__magic_name__   )
							args.append(__magic_name__   )
							return cls(*__magic_name__   )
