





'''simple docstring'''






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


snake_case_     =	logging.getLogger(__name__)
def 							_lowerCamelCase(  )		->  Optional[Any]:
    A       :       List[str]							=						argparse.ArgumentParser(
        description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''		)
    parser.add_argument(
        '''--dataset_name'''    ,       type=UpperCamelCase__    ,       default='''wikitext'''    ,       help='''Name of the training. Explore datasets at: hf.co/datasets.'''    ,       )
    parser.add_argument(
        '''--dataset_config'''    ,       type=UpperCamelCase__    ,       default='''wikitext-103-raw-v1'''    ,       help='''Configuration name of the dataset.'''		)
    parser.add_argument(
        '''--tokenizer_name_or_path'''    ,       type=UpperCamelCase__    ,       default='''sayakpaul/unigram-tokenizer-wikitext'''    ,       help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''    ,       )
    parser.add_argument(
        '''--shard_size'''    ,       type=UpperCamelCase__    ,       default=1_000    ,       help='''Number of entries to go in a single shard.'''    ,       )
    parser.add_argument('''--split'''    ,       type=UpperCamelCase__    ,       default='''train'''    ,       choices=['''train''', '''test''', '''validation''']		)
    parser.add_argument(
        '''--limit'''    ,       default=UpperCamelCase__    ,       type=UpperCamelCase__    ,       help='''Limit the number of shards (used for debugging).'''    ,       )
    parser.add_argument(
        '''--max_length'''    ,       type=UpperCamelCase__    ,       default=512    ,       help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
        ''' sequence length that is a multiple of 8.'''    ,       )
    parser.add_argument(
        '''--output_dir'''    ,       default='''tf-tpu'''    ,       type=UpperCamelCase__    ,       help='''Output directory where the TFRecord shards will be saved. If the'''
        ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
        ''' shards will be directly saved to a Google Cloud Storage bucket.'''    ,       )

    A       :       Any							=						parser.parse_args()
    return args
def 							_lowerCamelCase(  UpperCamelCase__    : Any		)		->  Dict:
    def fn(UpperCamelCase__    : Tuple		):
        return tokenizer(examples['''text''']		)

    return fn
def 							_lowerCamelCase(  UpperCamelCase__    : Dict		)		->  Tuple:
    A       :       str							=						[]
    for i in range(len(tokenized_data['''input_ids''']		)		):
        A       :       str							=						{
            '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]		)		),
            '''attention_mask''': tf.train.Feature(
                intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]		)		),
        }
        A       :       int							=						tf.train.Features(feature=UpperCamelCase__		)
        A       :       Any							=						tf.train.Example(features=UpperCamelCase__		)
        A       :       List[Any]							=						example.SerializeToString()
        records.append(UpperCamelCase__		)
    return records
def 							_lowerCamelCase(  UpperCamelCase__    : Dict		)		->  Tuple:
    A       :       Dict							=						datasets.load_dataset(args.dataset_name    ,       args.dataset_config    ,       split=args.split		)

    if args.limit is not None:
        A       :       int							=						min(len(UpperCamelCase__		)    ,       args.limit		)
        A       :       Optional[int]							=						dataset.select(range(UpperCamelCase__		)		)
        print(F'''Limiting the dataset to {args.limit} entries.'''		)

    A       :       List[str]							=						AutoTokenizer.from_pretrained(args.tokenizer_name_or_path		)

    # Handle output directory creation.
    # For serializing into a Google Cloud Storage Bucket, one needs to first
    # create a bucket.
    if "gs" not in args.output_dir:
        if not os.path.exists(args.output_dir		):
            os.makedirs(args.output_dir		)
        A       :       Dict							=						os.path.join(args.output_dir    ,       args.split		)
        if not os.path.exists(UpperCamelCase__		):
            os.makedirs(UpperCamelCase__		)
    else:
        A       :       Tuple							=						os.path.join(args.output_dir    ,       args.split		)

    # Tokenize the whole dataset at once.
    A       :       Union[str, Any]							=						tokenize_function(UpperCamelCase__		)
    A       :       int							=						dataset.map(UpperCamelCase__    ,       batched=UpperCamelCase__    ,       num_proc=4    ,       remove_columns=['''text''']		)

    # We need to concatenate all our texts together, and then split the result
    # into chunks of a fixed size, which we will call block_size. To do this, we
    # will use the map method again, with the option batched=True. When we use batched=True,
    # the function we pass to map() will be passed multiple inputs at once, allowing us
    # to group them into more or fewer examples than we had in the input.
    # This allows us to create our new fixed-length samples. The advantage of this
    # method is that we don't lose a whole lot of content from the dataset compared to the
    # case where we simply tokenize with a pre-defined max_length.

    def group_texts(UpperCamelCase__    : List[str]		):
        # Concatenate all texts.
        A       :       List[Any]							=						{k: sum(examples[k]    ,       []		) for k in examples.keys()}
        A       :       Tuple							=						len(concatenated_examples[list(examples.keys()		)[0]]		)
        # We drop the small remainder, though you could add padding instead if the model supports it
        # In this, as in all things, we advise you to follow your heart 🫀
        A       :       str							=						(total_length // args.max_length) * args.max_length
        # Split by chunks of max_len.
        A       :       Tuple							=						{
            k: [t[i : i + args.max_length] for i in range(0    ,       UpperCamelCase__    ,       args.max_length		)]
            for k, t in concatenated_examples.items()
        }
        return result

    A       :       Union[str, Any]							=						dataset_tokenized.map(UpperCamelCase__    ,       batched=UpperCamelCase__    ,       batch_size=1_000    ,       num_proc=4		)

    A       :       List[Any]							=						0
    A       :       List[Any]							=						0
    for shard in range(0    ,       len(UpperCamelCase__		)    ,       args.shard_size		):
        A       :       Dict							=						grouped_dataset[shard : shard + args.shard_size]
        A       :       Optional[Any]							=						len(dataset_snapshot['''input_ids''']		)
        A       :       Any							=						os.path.join(UpperCamelCase__    ,       F'''dataset-{shard_count}-{records_containing}.tfrecord'''		)
        A       :       List[str]							=						get_serialized_examples(UpperCamelCase__		)

        with tf.io.TFRecordWriter(UpperCamelCase__		) as out_file:
            for i in range(len(UpperCamelCase__		)		):
                A       :       Any							=						serialized_examples[i]
                out_file.write(UpperCamelCase__		)
            print('''Wrote file {} containing {} records'''.format(UpperCamelCase__    ,       UpperCamelCase__		)		)

        shard_count += 1
        total_records += records_containing

    with open(F'''split-{args.split}-records-count.txt'''    ,       '''w'''		) as f:
        print(F'''Total {args.split} records: {total_records}'''    ,       file=UpperCamelCase__		)


if __name__ == "__main__":
     snake_case_     =	parse_args()
     main(args)







'''simple docstring'''






import random
import unittest

import numpy as np

import transformers
from transformers import is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax


if is_flax_available():
     import os

     import jax.numpy as jnp
     from jax import jit

     from transformers import AutoTokenizer, FlaxAutoModelForCausalLM
     from transformers.modeling_flax_pytorch_utils import load_flax_weights_in_pytorch_model

     snake_case_     =	"""0.12"""  # assumed parallelism: 8


if is_torch_available():
     import torch
def 							_lowerCamelCase(  UpperCamelCase__    : int    ,       UpperCamelCase__    : Optional[int]    ,       UpperCamelCase__    : Any=None		)		->  List[Any]:
    if rng is None:
        A       :       int							=						random.Random()

    A       :       int							=						1
    for dim in shape:
        total_dims *= dim

    A       :       Union[str, Any]							=						[]
    for _ in range(UpperCamelCase__		):
        values.append(rng.randint(0    ,       vocab_size - 1		)		)

    A       :       Union[str, Any]							=						np.array(UpperCamelCase__    ,       dtype=jnp.intaa		).reshape(UpperCamelCase__		)

    return output
def 							_lowerCamelCase(  UpperCamelCase__    : Dict    ,       UpperCamelCase__    : Optional[Any]=None		)		->  Dict:
    A       :       Union[str, Any]							=						ids_tensor(UpperCamelCase__    ,       vocab_size=2    ,       rng=UpperCamelCase__		)
    # make sure that at least one token is attended to for each batch
    A       :       Any							=						1
    return attn_mask


@require_flax
class   _lowercase						:
   _UpperCamelCase       		=   None
   _UpperCamelCase       		=   ()


   def 					snake_case				( self					):
       A,	A       :       Optional[int]							=						self.model_tester.prepare_config_and_inputs_for_common()

       # cut to half length & take max batch_size 3
       A       :       Dict							=						2
       A       :       List[str]							=						inputs['''input_ids'''].shape[-1] // 2
       A       :       int							=						inputs['''input_ids'''][:max_batch_size, :sequence_length]

       A       :       Optional[Any]							=						jnp.ones_like(_UpperCAmelCase					)
       A       :       int							=						attention_mask[:max_batch_size, :sequence_length]

       # generate max 5 tokens
       A       :       Optional[int]							=						input_ids.shape[-1] + 5
       if config.eos_token_id is not None and config.pad_token_id is None:
           # hack to allow generate for models such as GPT2 as is done in `generate()`
           A       :       str							=						config.eos_token_id
       return config, input_ids, attention_mask, max_length


   @is_pt_flax_cross_test
   def 					snake_case				( self					):
       A,	A,	A,	A       :       Optional[Any]							=						self._get_input_ids_and_config()
       A       :       Optional[int]							=						False
       A       :       Union[str, Any]							=						max_length
       A       :       str							=						0

       for model_class in self.all_generative_model_classes:
           A       :       int							=						model_class(_UpperCAmelCase					)

           A       :       Dict							=						model_class.__name__[4:]  # Skip the "Flax" at the beginning
           A       :       int							=						getattr(_UpperCAmelCase    ,   _UpperCAmelCase					)
           A       :       Union[str, Any]							=						pt_model_class(_UpperCAmelCase					).eval()
           A       :       Tuple							=						load_flax_weights_in_pytorch_model(_UpperCAmelCase    ,   flax_model.params					)

           A       :       Union[str, Any]							=						flax_model.generate(_UpperCAmelCase					).sequences
           A       :       Optional[int]							=						pt_model.generate(torch.tensor(_UpperCAmelCase    ,   dtype=torch.long					)					)

           if flax_generation_outputs.shape[-1] > pt_generation_outputs.shape[-1]:
               A       :       List[Any]							=						flax_generation_outputs[:, : pt_generation_outputs.shape[-1]]

           self.assertListEqual(pt_generation_outputs.numpy().tolist()    ,   flax_generation_outputs.tolist()					)


   def 					snake_case				( self					):
       A,	A,	A,	A       :       str							=						self._get_input_ids_and_config()
       A       :       Optional[int]							=						False
       A       :       Dict							=						max_length

       for model_class in self.all_generative_model_classes:
           A       :       Any							=						model_class(_UpperCAmelCase					)

           A       :       Dict							=						model.generate(_UpperCAmelCase					).sequences
           self.assertEqual(generation_outputs.shape[-1]    ,   _UpperCAmelCase					)

           A       :       str							=						jit(model.generate					)
           A       :       List[Any]							=						jit_generate(_UpperCAmelCase					).sequences

           self.assertListEqual(generation_outputs.tolist()    ,   jit_generation_outputs.tolist()					)


   def 					snake_case				( self					):
       A,	A,	A,	A       :       List[Any]							=						self._get_input_ids_and_config()
       A       :       List[str]							=						True
       A       :       List[Any]							=						max_length

       for model_class in self.all_generative_model_classes:
           A       :       Any							=						model_class(_UpperCAmelCase					)

           A       :       Tuple							=						model.generate(_UpperCAmelCase					).sequences
           self.assertEqual(generation_outputs.shape[-1]    ,   _UpperCAmelCase					)

           A       :       Tuple							=						jit(model.generate					)
           A       :       int							=						jit_generate(_UpperCAmelCase					).sequences

           self.assertListEqual(generation_outputs.tolist()    ,   jit_generation_outputs.tolist()					)


   def 					snake_case				( self					):
       A,	A,	A,	A       :       List[str]							=						self._get_input_ids_and_config()
       A       :       Any							=						False
       A       :       str							=						max_length
       A       :       Optional[int]							=						2

       for model_class in self.all_generative_model_classes:
           A       :       Dict							=						model_class(_UpperCAmelCase					)

           A       :       Any							=						model.generate(_UpperCAmelCase					).sequences
           self.assertEqual(generation_outputs.shape[-1]    ,   _UpperCAmelCase					)

           A       :       List[Any]							=						jit(model.generate					)
           A       :       Optional[Any]							=						jit_generate(_UpperCAmelCase					).sequences

           self.assertListEqual(generation_outputs.tolist()    ,   jit_generation_outputs.tolist()					)


   def 					snake_case				( self					):
       A,	A,	A,	A       :       Optional[int]							=						self._get_input_ids_and_config()
       A       :       Dict							=						False
       A       :       List[Any]							=						max_length
       A       :       Optional[int]							=						2
       A       :       int							=						2

       for model_class in self.all_generative_model_classes:
           A       :       List[str]							=						model_class(_UpperCAmelCase					)

           A       :       List[str]							=						model.generate(_UpperCAmelCase					).sequences
           self.assertEqual(generation_outputs.shape[0]    ,   input_ids.shape[0] * config.num_return_sequences					)


   def 					snake_case				( self					):
       A,	A,	A,	A       :       List[Any]							=						self._get_input_ids_and_config()
       A       :       Any							=						True
       A       :       Any							=						max_length
       A       :       Any							=						0.8
       A       :       List[Any]							=						10
       A       :       List[Any]							=						0.3
       A       :       Union[str, Any]							=						1
       A       :       Any							=						8
       A       :       List[Any]							=						9

       for model_class in self.all_generative_model_classes:
           A       :       List[Any]							=						model_class(_UpperCAmelCase					)

           A       :       Any							=						model.generate(_UpperCAmelCase					).sequences
           self.assertEqual(generation_outputs.shape[-1]    ,   _UpperCAmelCase					)

           A       :       int							=						jit(model.generate					)
           A       :       List[Any]							=						jit_generate(_UpperCAmelCase					).sequences

           self.assertListEqual(generation_outputs.tolist()    ,   jit_generation_outputs.tolist()					)


   def 					snake_case				( self					):
       A,	A,	A,	A       :       Optional[Any]							=						self._get_input_ids_and_config()
       A       :       int							=						max_length
       A       :       Dict							=						1
       A       :       Tuple							=						8
       A       :       Optional[Any]							=						9

       for model_class in self.all_generative_model_classes:
           A       :       Dict							=						model_class(_UpperCAmelCase					)

           A       :       str							=						model.generate(_UpperCAmelCase					).sequences
           self.assertEqual(generation_outputs.shape[-1]    ,   _UpperCAmelCase					)

           A       :       int							=						jit(model.generate					)
           A       :       Optional[int]							=						jit_generate(_UpperCAmelCase					).sequences

           self.assertListEqual(generation_outputs.tolist()    ,   jit_generation_outputs.tolist()					)


   def 					snake_case				( self					):
       A,	A,	A,	A       :       int							=						self._get_input_ids_and_config()
       A       :       List[Any]							=						max_length
       A       :       Optional[Any]							=						2
       A       :       Tuple							=						1
       A       :       List[Any]							=						8
       A       :       Dict							=						9

       for model_class in self.all_generative_model_classes:
           A       :       Union[str, Any]							=						model_class(_UpperCAmelCase					)

           A       :       Any							=						model.generate(_UpperCAmelCase					).sequences
           self.assertEqual(generation_outputs.shape[-1]    ,   _UpperCAmelCase					)

           A       :       Union[str, Any]							=						jit(model.generate					)
           A       :       int							=						jit_generate(_UpperCAmelCase					).sequences

           self.assertListEqual(generation_outputs.tolist()    ,   jit_generation_outputs.tolist()					)


   def 					snake_case				( self					):
       A,	A,	A,	A       :       Dict							=						self._get_input_ids_and_config()

       # pad attention mask on the left
       A       :       Union[str, Any]							=						attention_mask.at[(0, 0)].set(0					)

       A       :       int							=						False
       A       :       List[Any]							=						max_length

       for model_class in self.all_generative_model_classes:
           A       :       List[str]							=						model_class(_UpperCAmelCase					)

           A       :       Dict							=						model.generate(_UpperCAmelCase    ,   attention_mask=_UpperCAmelCase					).sequences
           self.assertEqual(generation_outputs.shape[-1]    ,   _UpperCAmelCase					)

           A       :       Dict							=						jit(model.generate					)
           A       :       Optional[Any]							=						jit_generate(_UpperCAmelCase    ,   attention_mask=_UpperCAmelCase					).sequences

           self.assertListEqual(generation_outputs.tolist()    ,   jit_generation_outputs.tolist()					)


   def 					snake_case				( self					):
       A,	A,	A,	A       :       Optional[Any]							=						self._get_input_ids_and_config()

       # pad attention mask on the left
       A       :       Optional[Any]							=						attention_mask.at[(0, 0)].set(0					)

       A       :       Optional[int]							=						True
       A       :       int							=						max_length

       for model_class in self.all_generative_model_classes:
           A       :       int							=						model_class(_UpperCAmelCase					)

           A       :       Union[str, Any]							=						model.generate(_UpperCAmelCase    ,   attention_mask=_UpperCAmelCase					).sequences
           self.assertEqual(generation_outputs.shape[-1]    ,   _UpperCAmelCase					)

           A       :       Optional[Any]							=						jit(model.generate					)
           A       :       Dict							=						jit_generate(_UpperCAmelCase    ,   attention_mask=_UpperCAmelCase					).sequences

           self.assertListEqual(generation_outputs.tolist()    ,   jit_generation_outputs.tolist()					)


   def 					snake_case				( self					):
       A,	A,	A,	A       :       Optional[Any]							=						self._get_input_ids_and_config()

       # pad attention mask on the left
       A       :       Optional[int]							=						attention_mask.at[(0, 0)].set(0					)

       A       :       Any							=						2
       A       :       Any							=						max_length

       for model_class in self.all_generative_model_classes:
           A       :       Dict							=						model_class(_UpperCAmelCase					)

           A       :       Optional[Any]							=						model.generate(_UpperCAmelCase    ,   attention_mask=_UpperCAmelCase					).sequences
           self.assertEqual(generation_outputs.shape[-1]    ,   _UpperCAmelCase					)

           A       :       str							=						jit(model.generate					)
           A       :       List[str]							=						jit_generate(_UpperCAmelCase    ,   attention_mask=_UpperCAmelCase					).sequences

           self.assertListEqual(generation_outputs.tolist()    ,   jit_generation_outputs.tolist()					)







@require_flax
class   _lowercase						(							unittest.TestCase				):


   def 					snake_case				( self					):
       A       :       List[Any]							=						AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-bert'''					)
       A       :       Dict							=						FlaxAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''					)

       A       :       Union[str, Any]							=						'''Hello world'''
       A       :       Union[str, Any]							=						tokenizer(_UpperCAmelCase    ,   return_tensors='''np'''					).input_ids

       # typos are quickly detected (the correct argument is `do_sample`)
       with self.assertRaisesRegex(_UpperCAmelCase    ,   '''do_samples'''					):
           model.generate(_UpperCAmelCase    ,   do_samples=_UpperCAmelCase					)

       # arbitrary arguments that will not be used anywhere are also not accepted
       with self.assertRaisesRegex(_UpperCAmelCase    ,   '''foo'''					):
           A       :       List[Any]							=						{'''foo''': '''bar'''}
           model.generate(_UpperCAmelCase    ,   **_UpperCAmelCase					)
