





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCamelCase__      =logging.getLogger(__name__)






def     lowerCamelCase__			():
     _SCREAMING_SNAKE_CASE       :	str						    =							argparse.ArgumentParser(
         description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."	)
     parser.add_argument(
         "--dataset_name",       type=__lowerCamelCase,       default="wikitext",       help="Name of the training. Explore datasets at: hf.co/datasets.",       )
     parser.add_argument(
         "--dataset_config",       type=__lowerCamelCase,       default="wikitext-103-raw-v1",       help="Configuration name of the dataset."	)
     parser.add_argument(
         "--tokenizer_name_or_path",       type=__lowerCamelCase,       default="sayakpaul/unigram-tokenizer-wikitext",       help="Tokenizer identifier. Can be a local filepath or a Hub identifier.",       )
     parser.add_argument(
         "--shard_size",       type=__lowerCamelCase,       default=1000,       help="Number of entries to go in a single shard.",       )
     parser.add_argument("--split",       type=__lowerCamelCase,       default="train",       choices=["train", "test", "validation"]	)
     parser.add_argument(
         "--limit",       default=__lowerCamelCase,       type=__lowerCamelCase,       help="Limit the number of shards (used for debugging).",       )
     parser.add_argument(
         "--max_length",       type=__lowerCamelCase,       default=512,       help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
         " sequence length that is a multiple of 8.",       )
     parser.add_argument(
         "--output_dir",       default="tf-tpu",       type=__lowerCamelCase,       help="Output directory where the TFRecord shards will be saved. If the"
         " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
         " shards will be directly saved to a Google Cloud Storage bucket.",       )

     _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							parser.parse_args()
     return args






def     lowerCamelCase__			(__lowerCamelCase	):
     def fn(__lowerCamelCase	):
          return tokenizer(examples["text"]	)

     return fn






def     lowerCamelCase__			(__lowerCamelCase	):
     _SCREAMING_SNAKE_CASE       :	Any						    =							[]
     for i in range(len(tokenized_data["input_ids"]	)	):
          _SCREAMING_SNAKE_CASE       :	Dict						    =							{
              "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]	)	),
              "attention_mask": tf.train.Feature(
                  intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]	)	),
          }
          _SCREAMING_SNAKE_CASE       :	List[str]						    =							tf.train.Features(feature=__lowerCamelCase	)
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							tf.train.Example(features=__lowerCamelCase	)
          _SCREAMING_SNAKE_CASE       :	Tuple						    =							example.SerializeToString()
          records.append(__lowerCamelCase	)
     return records






def     lowerCamelCase__			(__lowerCamelCase	):
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							datasets.load_dataset(args.dataset_name,       args.dataset_config,       split=args.split	)

     if args.limit is not None:
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							min(len(__lowerCamelCase	),       args.limit	)
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							dataset.select(range(__lowerCamelCase	)	)
          print(f"""Limiting the dataset to {args.limit} entries."""	)

     _SCREAMING_SNAKE_CASE       :	str						    =							AutoTokenizer.from_pretrained(args.tokenizer_name_or_path	)

     # Handle output directory creation.
     # For serializing into a Google Cloud Storage Bucket, one needs to first
     # create a bucket.
     if "gs" not in args.output_dir:
          if not os.path.exists(args.output_dir	):
               os.makedirs(args.output_dir	)
          _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							os.path.join(args.output_dir,       args.split	)
          if not os.path.exists(__lowerCamelCase	):
               os.makedirs(__lowerCamelCase	)
     else:
          _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(args.output_dir,       args.split	)

     # Tokenize the whole dataset at once.
     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							tokenize_function(__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							dataset.map(__lowerCamelCase,       batched=__lowerCamelCase,       num_proc=4,       remove_columns=["text"]	)

     # We need to concatenate all our texts together, and then split the result
     # into chunks of a fixed size, which we will call block_size. To do this, we
     # will use the map method again, with the option batched=True. When we use batched=True,
     # the function we pass to map() will be passed multiple inputs at once, allowing us
     # to group them into more or fewer examples than we had in the input.
     # This allows us to create our new fixed-length samples. The advantage of this
     # method is that we don't lose a whole lot of content from the dataset compared to the
     # case where we simply tokenize with a pre-defined max_length.

     def group_texts(__lowerCamelCase	):
          # Concatenate all texts.
          _SCREAMING_SNAKE_CASE       :	Dict						    =							{k: sum(examples[k],       []	) for k in examples.keys()}
          _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							len(concatenated_examples[list(examples.keys()	)[0]]	)
          # We drop the small remainder, though you could add padding instead if the model supports it
          # In this, as in all things, we advise you to follow your heart 🫀
          _SCREAMING_SNAKE_CASE       :	Dict						    =							(total_length // args.max_length) * args.max_length
          # Split by chunks of max_len.
          _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							{
              k: [t[i : i + args.max_length] for i in range(0,       __lowerCamelCase,       args.max_length	)]
              for k, t in concatenated_examples.items()
          }
          return result

     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							dataset_tokenized.map(__lowerCamelCase,       batched=__lowerCamelCase,       batch_size=1000,       num_proc=4	)

     _SCREAMING_SNAKE_CASE       :	Tuple						    =							0
     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							0
     for shard in range(0,       len(__lowerCamelCase	),       args.shard_size	):
          _SCREAMING_SNAKE_CASE       :	int						    =							grouped_dataset[shard : shard + args.shard_size]
          _SCREAMING_SNAKE_CASE       :	Tuple						    =							len(dataset_snapshot["input_ids"]	)
          _SCREAMING_SNAKE_CASE       :	Dict						    =							os.path.join(__lowerCamelCase,       f"""dataset-{shard_count}-{records_containing}.tfrecord"""	)
          _SCREAMING_SNAKE_CASE       :	int						    =							get_serialized_examples(__lowerCamelCase	)

          with tf.io.TFRecordWriter(__lowerCamelCase	) as out_file:
               for i in range(len(__lowerCamelCase	)	):
                    _SCREAMING_SNAKE_CASE       :	Tuple						    =							serialized_examples[i]
                    out_file.write(__lowerCamelCase	)
               print("Wrote file {} containing {} records".format(__lowerCamelCase,       __lowerCamelCase	)	)

          shard_count += 1
          total_records += records_containing

     with open(f"""split-{args.split}-records-count.txt""",       "w"	) as f:
          print(f"""Total {args.split} records: {total_records}""",       file=__lowerCamelCase	)


if __name__ == "__main__":
       UpperCamelCase__      =parse_args()
       main(args)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__      =logging.get_logger(__name__)

UpperCamelCase__      ={
    'facebook/s2t-wav2vec2-large-en-de': (
        'https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/config.json'
    ),
    # See all Speech2Text models at https://huggingface.co/models?filter=speech2text2
}





class 			lowerCAmelCase__( __lowercase    ):






       '''simple docstring'''


       __snake_case 			=    'speech_to_text_2'
       __snake_case 			=    ['past_key_values']
       __snake_case 			=    {'num_attention_heads': 'decoder_attention_heads', 'hidden_size': 'd_model'}




       def __init__(      self    ,						__lowerCamelCase=1_0_0_0_0    ,						__lowerCamelCase=6    ,						__lowerCamelCase=2_0_4_8    ,						__lowerCamelCase=4    ,						__lowerCamelCase=0.0    ,						__lowerCamelCase=True    ,						__lowerCamelCase="relu"    ,						__lowerCamelCase=2_5_6    ,						__lowerCamelCase=0.1    ,						__lowerCamelCase=0.0    ,						__lowerCamelCase=0.0    ,						__lowerCamelCase=0.02    ,						__lowerCamelCase=2    ,						__lowerCamelCase=True    ,						__lowerCamelCase=1    ,						__lowerCamelCase=0    ,						__lowerCamelCase=2    ,						__lowerCamelCase=1_0_2_4    ,						**__lowerCamelCase    ,						)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							vocab_size
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							d_model
            _SCREAMING_SNAKE_CASE       :	Dict						    =							decoder_ffn_dim
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							decoder_layers
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							decoder_attention_heads
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							dropout
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							attention_dropout
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							activation_dropout
            _SCREAMING_SNAKE_CASE       :	Dict						    =							activation_function
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							init_std
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							decoder_layerdrop
            _SCREAMING_SNAKE_CASE       :	int						    =							use_cache
            _SCREAMING_SNAKE_CASE       :	int						    =							decoder_layers
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							scale_embedding  # scale factor will be sqrt(d_model) if True
            _SCREAMING_SNAKE_CASE       :	str						    =							max_target_positions

            super().__init__(
                pad_token_id=__lowerCamelCase    ,						bos_token_id=__lowerCamelCase    ,						eos_token_id=__lowerCamelCase    ,						decoder_start_token_id=__lowerCamelCase    ,						**__lowerCamelCase    ,						)