
'''simple docstring'''
import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCAmelCase					:    List[str]           = logging.getLogger(__name__)







def 		A_(     ):
							UpperCamelCase   			=     argparse.ArgumentParser(
							    description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.')
							parser.add_argument(
							    '--dataset_name'		,     type=A		,     default='wikitext'		,     help='Name of the training. Explore datasets at: hf.co/datasets.'		,     )
							parser.add_argument(
							    '--dataset_config'		,     type=A		,     default='wikitext-103-raw-v1'		,     help='Configuration name of the dataset.')
							parser.add_argument(
							    '--tokenizer_name_or_path'		,     type=A		,     default='sayakpaul/unigram-tokenizer-wikitext'		,     help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'		,     )
							parser.add_argument(
							    '--shard_size'		,     type=A		,     default=1000		,     help='Number of entries to go in a single shard.'		,     )
							parser.add_argument('--split'		,     type=A		,     default='train'		,     choices=['train', 'test', 'validation'])
							parser.add_argument(
							    '--limit'		,     default=A		,     type=A		,     help='Limit the number of shards (used for debugging).'		,     )
							parser.add_argument(
							    '--max_length'		,     type=A		,     default=512		,     help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
							    ' sequence length that is a multiple of 8.'		,     )
							parser.add_argument(
							    '--output_dir'		,     default='tf-tpu'		,     type=A		,     help='Output directory where the TFRecord shards will be saved. If the'
							    ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
							    ' shards will be directly saved to a Google Cloud Storage bucket.'		,     )

							UpperCamelCase   			=     parser.parse_args()
							return args







def 		A_(     A						:		Any):
							def fn(A						:		Union[str, Any]):
														return tokenizer(examples['text'])

							return fn







def 		A_(     A						:		Optional[Any]):
							UpperCamelCase   			=     []
							for i in range(len(tokenized_data['input_ids'])):
														UpperCamelCase   			=     {
														    'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i])),
														    'attention_mask': tf.train.Feature(
														        intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i])),
														}
														UpperCamelCase   			=     tf.train.Features(feature=A)
														UpperCamelCase   			=     tf.train.Example(features=A)
														UpperCamelCase   			=     example.SerializeToString()
														records.append(A)
							return records







def 		A_(     A						:		List[Any]):
							UpperCamelCase   			=     datasets.load_dataset(args.dataset_name		,     args.dataset_config		,     split=args.split)

							if args.limit is not None:
														UpperCamelCase   			=     min(len(A)		,     args.limit)
														UpperCamelCase   			=     dataset.select(range(A))
														print(f'''Limiting the dataset to {args.limit} entries.''')

							UpperCamelCase   			=     AutoTokenizer.from_pretrained(args.tokenizer_name_or_path)

							# Handle output directory creation.
							# For serializing into a Google Cloud Storage Bucket, one needs to first
							# create a bucket.
							if "gs" not in args.output_dir:
														if not os.path.exists(args.output_dir):
																					os.makedirs(args.output_dir)
														UpperCamelCase   			=     os.path.join(args.output_dir		,     args.split)
														if not os.path.exists(A):
																					os.makedirs(A)
							else:
														UpperCamelCase   			=     os.path.join(args.output_dir		,     args.split)

							# Tokenize the whole dataset at once.
							UpperCamelCase   			=     tokenize_function(A)
							UpperCamelCase   			=     dataset.map(A		,     batched=A		,     num_proc=4		,     remove_columns=['text'])

							# We need to concatenate all our texts together, and then split the result
							# into chunks of a fixed size, which we will call block_size. To do this, we
							# will use the map method again, with the option batched=True. When we use batched=True,
							# the function we pass to map() will be passed multiple inputs at once, allowing us
							# to group them into more or fewer examples than we had in the input.
							# This allows us to create our new fixed-length samples. The advantage of this
							# method is that we don't lose a whole lot of content from the dataset compared to the
							# case where we simply tokenize with a pre-defined max_length.

							def group_texts(A						:		int):
														# Concatenate all texts.
														UpperCamelCase   			=     {k: sum(examples[k]		,     []) for k in examples.keys()}
														UpperCamelCase   			=     len(concatenated_examples[list(examples.keys())[0]])
														# We drop the small remainder, though you could add padding instead if the model supports it
														# In this, as in all things, we advise you to follow your heart 🫀
														UpperCamelCase   			=     (total_length // args.max_length) * args.max_length
														# Split by chunks of max_len.
														UpperCamelCase   			=     {
														    k: [t[i : i + args.max_length] for i in range(0		,     A		,     args.max_length)]
														    for k, t in concatenated_examples.items()
														}
														return result

							UpperCamelCase   			=     dataset_tokenized.map(A		,     batched=A		,     batch_size=1000		,     num_proc=4)

							UpperCamelCase   			=     0
							UpperCamelCase   			=     0
							for shard in range(0		,     len(A)		,     args.shard_size):
														UpperCamelCase   			=     grouped_dataset[shard : shard + args.shard_size]
														UpperCamelCase   			=     len(dataset_snapshot['input_ids'])
														UpperCamelCase   			=     os.path.join(A		,     f'''dataset-{shard_count}-{records_containing}.tfrecord''')
														UpperCamelCase   			=     get_serialized_examples(A)

														with tf.io.TFRecordWriter(A) as out_file:
																					for i in range(len(A)):
																												UpperCamelCase   			=     serialized_examples[i]
																												out_file.write(A)
																					print('Wrote file {} containing {} records'.format(A		,     A))

														shard_count += 1
														total_records += records_containing

							with open(f'''split-{args.split}-records-count.txt'''		,     'w') as f:
														print(f'''Total {args.split} records: {total_records}'''		,     file=A)


if __name__ == "__main__":
						lowerCAmelCase					:    List[str]           = parse_args()
						main(args)






'''simple docstring'''
import enum
import warnings

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
						import tensorflow as tf

						from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

if is_torch_available():
						from ..models.auto.modeling_auto import MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

lowerCAmelCase					:    Dict           = logging.get_logger(__name__)


class 			SCREAMING_SNAKE_CASE__      (     enum.Enum):
				lowerCAmelCase_     			=							0
				lowerCAmelCase_     			=							1


@add_end_docstrings(snake_case_)
class 			SCREAMING_SNAKE_CASE__      (     snake_case_):
				lowerCAmelCase_     			=							"""generated"""







				def __init__(       self						,				*A_						,				**A_     )->		Optional[int]:







											'''simple docstring'''






											super().__init__(*A_						,				**A_     )

											self.check_model_type(
											    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
											    if self.framework == 'tf'
											    else MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING     )







				def 						UpperCAmelCase_    (       self						,				A_=None						,				A_=None						,				A_=None						,				A_=None						,				A_=None						,				A_=None						,				**A_						,				)->		Optional[Any]:







											'''simple docstring'''






											UpperCamelCase   			=     {}
											if truncation is not None:
																		UpperCamelCase   			=     truncation

											UpperCamelCase   			=     generate_kwargs

											UpperCamelCase   			=     {}
											if return_tensors is not None and return_type is None:
																		UpperCamelCase   			=     ReturnType.TENSORS if return_tensors else ReturnType.TEXT
											if return_type is not None:
																		UpperCamelCase   			=     return_type

											if clean_up_tokenization_spaces is not None:
																		UpperCamelCase   			=     clean_up_tokenization_spaces

											if stop_sequence is not None:
																		UpperCamelCase   			=     self.tokenizer.encode(A_						,				add_special_tokens=A_     )
																		if len(A_     ) > 1:
																									warnings.warn(
																									    'Stopping on a multiple token sequence is not yet supported on transformers. The first token of'
																									    ' the stop sequence will be used as the stop sequence string in the interim.'     )
																		UpperCamelCase   			=     stop_sequence_ids[0]

											return preprocess_params, forward_params, postprocess_params







				def 						UpperCAmelCase_    (       self						,				A_						,				A_						,				A_     )->		Optional[int]:







											'''simple docstring'''






											return True







				def 						UpperCAmelCase_    (       self						,				*A_						,				A_     )->		Any:







											'''simple docstring'''






											UpperCamelCase   			=     self.model.config.prefix if self.model.config.prefix is not None else ''
											if isinstance(args[0]						,				A_     ):
																		if self.tokenizer.pad_token_id is None:
																									raise ValueError('Please make sure that the tokenizer has a pad_token_id when using a batch input'     )
																		UpperCamelCase   			=     ([prefix + arg for arg in args[0]],)
																		UpperCamelCase   			=     True

											elif isinstance(args[0]						,				A_     ):
																		UpperCamelCase   			=     (prefix + args[0],)
																		UpperCamelCase   			=     False
											else:
																		raise ValueError(
																		    F''' `args[0]`: {args[0]} have the wrong format. The should be either of type `str` or type `list`'''     )
											UpperCamelCase   			=     self.tokenizer(*A_						,				padding=A_						,				truncation=A_						,				return_tensors=self.framework     )
											# This is produced by tokenizers but is an invalid generate kwargs
											if "token_type_ids" in inputs:
																		del inputs["token_type_ids"]
											return inputs







				def __call__(       self						,				*A_						,				**A_     )->		Union[str, Any]:







											'''simple docstring'''






											UpperCamelCase   			=     super().__call__(*A_						,				**A_     )
											if (
											    isinstance(args[0]						,				A_     )
											    and all(isinstance(A_						,				A_     ) for el in args[0]     )
											    and all(len(A_     ) == 1 for res in result     )
											):
																		return [res[0] for res in result]
											return result







				def 						UpperCAmelCase_    (       self						,				A_						,				A_=TruncationStrategy.DO_NOT_TRUNCATE						,				**A_     )->		Any:







											'''simple docstring'''






											UpperCamelCase   			=     self._parse_and_tokenize(A_						,				truncation=A_						,				**A_     )
											return inputs







				def 						UpperCAmelCase_    (       self						,				A_						,				**A_     )->		int:







											'''simple docstring'''






											if self.framework == "pt":
																		UpperCamelCase			,	UpperCamelCase   			=     model_inputs['input_ids'].shape
											elif self.framework == "tf":
																		UpperCamelCase			,	UpperCamelCase   			=     tf.shape(model_inputs['input_ids']     ).numpy()

											UpperCamelCase   			=     generate_kwargs.get('min_length'						,				self.model.config.min_length     )
											UpperCamelCase   			=     generate_kwargs.get('max_length'						,				self.model.config.max_length     )
											self.check_inputs(A_						,				generate_kwargs['min_length']						,				generate_kwargs['max_length']     )
											UpperCamelCase   			=     self.model.generate(**A_						,				**A_     )
											UpperCamelCase   			=     output_ids.shape[0]
											if self.framework == "pt":
																		UpperCamelCase   			=     output_ids.reshape(A_						,				out_b // in_b						,				*output_ids.shape[1:]     )
											elif self.framework == "tf":
																		UpperCamelCase   			=     tf.reshape(A_						,				(in_b, out_b // in_b, *output_ids.shape[1:])     )
											return {"output_ids": output_ids}







				def 						UpperCAmelCase_    (       self						,				A_						,				A_=ReturnType.TEXT						,				A_=False     )->		Optional[Any]:







											'''simple docstring'''






											UpperCamelCase   			=     []
											for output_ids in model_outputs["output_ids"][0]:
																		if return_type == ReturnType.TENSORS:
																									UpperCamelCase   			=     {F'''{self.return_name}_token_ids''': output_ids}
																		elif return_type == ReturnType.TEXT:
																									UpperCamelCase   			=     {
																									    F'''{self.return_name}_text''': self.tokenizer.decode(
																									        A_						,				skip_special_tokens=A_						,				clean_up_tokenization_spaces=A_						,				)
																									}
																		records.append(A_     )
											return records


@add_end_docstrings(snake_case_)
class 			SCREAMING_SNAKE_CASE__      (     snake_case_):
				lowerCAmelCase_     			=							"""summary"""







				def __call__(       self						,				*A_						,				**A_     )->		Optional[int]:







											'''simple docstring'''






											return super().__call__(*A_						,				**A_     )







				def 						UpperCAmelCase_    (       self						,				A_						,				A_						,				A_     )->		bool:







											'''simple docstring'''






											if max_length < min_length:
																		logger.warning(F'''Your min_length={min_length} must be inferior than your max_length={max_length}.'''     )

											if input_length < max_length:
																		logger.warning(
																		    F'''Your max_length is set to {max_length}, but your input_length is only {input_length}. Since this is '''
																		    'a summarization task, where outputs shorter than the input are typically wanted, you might '
																		    F'''consider decreasing max_length manually, e.g. summarizer(\'...\', max_length={input_length//2})'''     )


@add_end_docstrings(snake_case_)
class 			SCREAMING_SNAKE_CASE__      (     snake_case_):
				lowerCAmelCase_     			=							"""translation"""







				def 						UpperCAmelCase_    (       self						,				A_						,				A_						,				A_     )->		List[Any]:







											'''simple docstring'''






											if input_length > 0.9 * max_length:
																		logger.warning(
																		    F'''Your input_length: {input_length} is bigger than 0.9 * max_length: {max_length}. You might consider '''
																		    'increasing your max_length manually, e.g. translator(\'...\', max_length=400)'     )
											return True







				def 						UpperCAmelCase_    (       self						,				*A_						,				A_=TruncationStrategy.DO_NOT_TRUNCATE						,				A_=None						,				A_=None     )->		Dict:







											'''simple docstring'''






											if getattr(self.tokenizer						,				'_build_translation_inputs'						,				A_     ):
																		return self.tokenizer._build_translation_inputs(
																		    *A_						,				return_tensors=self.framework						,				truncation=A_						,				src_lang=A_						,				tgt_lang=A_     )
											else:
																		return super()._parse_and_tokenize(*A_						,				truncation=A_     )







				def 						UpperCAmelCase_    (       self						,				A_=None						,				A_=None						,				**A_     )->		str:







											'''simple docstring'''






											UpperCamelCase			,	UpperCamelCase			,	UpperCamelCase   			=     super()._sanitize_parameters(**A_     )
											if src_lang is not None:
																		UpperCamelCase   			=     src_lang
											if tgt_lang is not None:
																		UpperCamelCase   			=     tgt_lang
											if src_lang is None and tgt_lang is None:
																		# Backward compatibility, direct arguments use is preferred.
																		UpperCamelCase   			=     kwargs.get('task'						,				self.task     )
																		UpperCamelCase   			=     task.split('_'     )
																		if task and len(A_     ) == 4:
																									# translation, XX, to YY
																									UpperCamelCase   			=     items[1]
																									UpperCamelCase   			=     items[3]
											return preprocess_params, forward_params, postprocess_params







				def __call__(       self						,				*A_						,				**A_     )->		Any:







											'''simple docstring'''






											return super().__call__(*A_						,				**A_     )




