

import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class      SCREAMING_SNAKE_CASE__  (    _lowerCAmelCase			,						unittest.TestCase		):





       '''simple docstring'''




       UpperCamelCase__			:  str					     =							FunnelTokenizer
       UpperCamelCase__			:  List[str]					     =							FunnelTokenizerFast
       UpperCamelCase__			:  Any					     =							True
       UpperCamelCase__			:  Any					     =							True

       def UpperCAmelCase_    (    self							:    List[Any]				)      ->     int:
             super().setUp()

             snake_case__											= [
                 """<unk>""",
                 """<cls>""",
                 """<sep>""",
                 """want""",
                 """##want""",
                 """##ed""",
                 """wa""",
                 """un""",
                 """runn""",
                 """##ing""",
                 """,""",
                 """low""",
                 """lowest""",
             ]
             snake_case__											= os.path.join(self.tmpdirname     ,		VOCAB_FILES_NAMES["""vocab_file"""]				)
             with open(self.vocab_file     ,		"""w"""     ,		encoding="""utf-8"""				) as vocab_writer:
                   vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]				)				)

       def UpperCAmelCase_    (    self							:    List[Any]     ,		**lowerCAmelCase__							:    Union[str, Any]				)      ->     Union[str, Any]:
             return FunnelTokenizer.from_pretrained(self.tmpdirname     ,		**UpperCAmelCase_				)

       def UpperCAmelCase_    (    self							:    Tuple     ,		**lowerCAmelCase__							:    Tuple				)      ->     List[Any]:
             return FunnelTokenizerFast.from_pretrained(self.tmpdirname     ,		**UpperCAmelCase_				)

       def UpperCAmelCase_    (    self							:    Optional[int]     ,		lowerCAmelCase__							:    int				)      ->     Dict:
             snake_case__											= """UNwant\u00E9d,running"""
             snake_case__											= """unwanted, running"""
             return input_text, output_text

       def UpperCAmelCase_    (    self							:    Tuple				)      ->     Any:
             snake_case__											= self.tokenizer_class(self.vocab_file				)

             snake_case__											= tokenizer.tokenize("""UNwant\u00E9d,running"""				)
             self.assertListEqual(UpperCAmelCase_     ,		["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]				)
             self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_				)     ,		[7, 4, 5, 10, 8, 9]				)

       def UpperCAmelCase_    (    self							:    str				)      ->     int:
             snake_case__											= self.get_tokenizers(do_lower_case=UpperCAmelCase_				)
             for tokenizer in tokenizers:
                   snake_case__											= tokenizer("""UNwant\u00E9d,running"""				)
                   snake_case__											= len(inputs["""input_ids"""]				) - 1
                   self.assertListEqual(inputs["""token_type_ids"""]     ,		[2] + [0] * sentence_len				)

                   snake_case__											= tokenizer("""UNwant\u00E9d,running"""     ,		"""UNwant\u00E9d,running"""				)
                   self.assertListEqual(inputs["""token_type_ids"""]     ,		[2] + [0] * sentence_len + [1] * sentence_len				)








"""simple docstring"""


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


SCREAMING_SNAKE_CASE			  =  logging.getLogger(__name__)



def 						lowerCamelCase__      (				)->				Tuple:







       """simple docstring"""
       UpperCamelCase       			=		argparse.ArgumentParser(
           description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."					)
       parser.add_argument(
           "--dataset_name"		,   type=UpperCAmelCase_		,   default="wikitext"		,   help="Name of the training. Explore datasets at: hf.co/datasets."		,   )
       parser.add_argument(
           "--dataset_config"		,   type=UpperCAmelCase_		,   default="wikitext-103-raw-v1"		,   help="Configuration name of the dataset."					)
       parser.add_argument(
           "--tokenizer_name_or_path"		,   type=UpperCAmelCase_		,   default="sayakpaul/unigram-tokenizer-wikitext"		,   help="Tokenizer identifier. Can be a local filepath or a Hub identifier."		,   )
       parser.add_argument(
           "--shard_size"		,   type=UpperCAmelCase_		,   default=10_00		,   help="Number of entries to go in a single shard."		,   )
       parser.add_argument("--split"		,   type=UpperCAmelCase_		,   default="train"		,   choices=["train", "test", "validation"]					)
       parser.add_argument(
           "--limit"		,   default=UpperCAmelCase_		,   type=UpperCAmelCase_		,   help="Limit the number of shards (used for debugging)."		,   )
       parser.add_argument(
           "--max_length"		,   type=UpperCAmelCase_		,   default=5_12		,   help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
           " sequence length that is a multiple of 8."		,   )
       parser.add_argument(
           "--output_dir"		,   default="tf-tpu"		,   type=UpperCAmelCase_		,   help="Output directory where the TFRecord shards will be saved. If the"
           " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
           " shards will be directly saved to a Google Cloud Storage bucket."		,   )

       UpperCamelCase       			=		parser.parse_args()
       return args



def 						lowerCamelCase__      (				UpperCAmelCase_					)->				Optional[int]:







       """simple docstring"""
       def fn(UpperCAmelCase_					):
              return tokenizer(examples["text"]					)

       return fn



def 						lowerCamelCase__      (				UpperCAmelCase_					)->				Union[str, Any]:







       """simple docstring"""
       UpperCamelCase       			=		[]
       for i in range(len(tokenized_data["input_ids"]					)					):
              UpperCamelCase       			=		{
                  "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]					)					),
                  "attention_mask": tf.train.Feature(
                      intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]					)					),
              }
              UpperCamelCase       			=		tf.train.Features(feature=UpperCAmelCase_					)
              UpperCamelCase       			=		tf.train.Example(features=UpperCAmelCase_					)
              UpperCamelCase       			=		example.SerializeToString()
              records.append(UpperCAmelCase_					)
       return records



def 						lowerCamelCase__      (				UpperCAmelCase_					)->				int:







       """simple docstring"""
       UpperCamelCase       			=		datasets.load_dataset(args.dataset_name		,   args.dataset_config		,   split=args.split					)

       if args.limit is not None:
              UpperCamelCase       			=		min(len(UpperCAmelCase_					)		,   args.limit					)
              UpperCamelCase       			=		dataset.select(range(UpperCAmelCase_					)					)
              print(F"Limiting the dataset to {args.limit} entries."					)

       UpperCamelCase       			=		AutoTokenizer.from_pretrained(args.tokenizer_name_or_path					)

       # Handle output directory creation.
       # For serializing into a Google Cloud Storage Bucket, one needs to first
       # create a bucket.
       if "gs" not in args.output_dir:
              if not os.path.exists(args.output_dir					):
                     os.makedirs(args.output_dir					)
              UpperCamelCase       			=		os.path.join(args.output_dir		,   args.split					)
              if not os.path.exists(UpperCAmelCase_					):
                     os.makedirs(UpperCAmelCase_					)
       else:
              UpperCamelCase       			=		os.path.join(args.output_dir		,   args.split					)

       # Tokenize the whole dataset at once.
       UpperCamelCase       			=		tokenize_function(UpperCAmelCase_					)
       UpperCamelCase       			=		dataset.map(UpperCAmelCase_		,   batched=UpperCAmelCase_		,   num_proc=4		,   remove_columns=["text"]					)

       # We need to concatenate all our texts together, and then split the result
       # into chunks of a fixed size, which we will call block_size. To do this, we
       # will use the map method again, with the option batched=True. When we use batched=True,
       # the function we pass to map() will be passed multiple inputs at once, allowing us
       # to group them into more or fewer examples than we had in the input.
       # This allows us to create our new fixed-length samples. The advantage of this
       # method is that we don't lose a whole lot of content from the dataset compared to the
       # case where we simply tokenize with a pre-defined max_length.

       def group_texts(UpperCAmelCase_					):
              # Concatenate all texts.
              UpperCamelCase       			=		{k: sum(examples[k]		,   []					) for k in examples.keys()}
              UpperCamelCase       			=		len(concatenated_examples[list(examples.keys()					)[0]]					)
              # We drop the small remainder, though you could add padding instead if the model supports it
              # In this, as in all things, we advise you to follow your heart 🫀
              UpperCamelCase       			=		(total_length // args.max_length) * args.max_length
              # Split by chunks of max_len.
              UpperCamelCase       			=		{
                  k: [t[i : i + args.max_length] for i in range(0		,   UpperCAmelCase_		,   args.max_length					)]
                  for k, t in concatenated_examples.items()
              }
              return result

       UpperCamelCase       			=		dataset_tokenized.map(UpperCAmelCase_		,   batched=UpperCAmelCase_		,   batch_size=10_00		,   num_proc=4					)

       UpperCamelCase       			=		0
       UpperCamelCase       			=		0
       for shard in range(0		,   len(UpperCAmelCase_					)		,   args.shard_size					):
              UpperCamelCase       			=		grouped_dataset[shard : shard + args.shard_size]
              UpperCamelCase       			=		len(dataset_snapshot["input_ids"]					)
              UpperCamelCase       			=		os.path.join(UpperCAmelCase_		,   F"dataset-{shard_count}-{records_containing}.tfrecord"					)
              UpperCamelCase       			=		get_serialized_examples(UpperCAmelCase_					)

              with tf.io.TFRecordWriter(UpperCAmelCase_					) as out_file:
                     for i in range(len(UpperCAmelCase_					)					):
                            UpperCamelCase       			=		serialized_examples[i]
                            out_file.write(UpperCAmelCase_					)
                     print("Wrote file {} containing {} records".format(UpperCAmelCase_		,   UpperCAmelCase_					)					)

              shard_count += 1
              total_records += records_containing

       with open(F"split-{args.split}-records-count.txt"		,   "w"					) as f:
              print(F"Total {args.split} records: {total_records}"		,   file=UpperCAmelCase_					)


if __name__ == "__main__":
      SCREAMING_SNAKE_CASE			  =  parse_args()
      main(args)





