




"""simple docstring"""

import argparse
import collections
import os
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
a							      =     '''src/transformers'''
a							      =     '''docs/source/en'''
a							      =     '''.'''






def 		_snake_case	(   _snake_case				:   Tuple							,    _snake_case				:   Dict							,    _snake_case				:   Tuple				)       ->   Optional[Any]:






      '''simple docstring'''


      with open(_snake_case							,    'r'							,    encoding='utf-8'							,    newline='\n'				) as f:
            _A     =  f.readlines()
      # Find the start prompt.
      _A     =  0
      while not lines[start_index].startswith(_snake_case				):
            start_index += 1
      start_index += 1

      _A     =  start_index
      while not lines[end_index].startswith(_snake_case				):
            end_index += 1
      end_index -= 1

      while len(lines[start_index]				) <= 1:
            start_index += 1
      while len(lines[end_index]				) <= 1:
            end_index -= 1
      end_index += 1
      return "".join(lines[start_index:end_index]				), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
a							      =     '''Model|Encoder|Decoder|ForConditionalGeneration'''
# Regexes that match TF/Flax/PT model names.
a							      =     re.compile(r'''TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
a							      =     re.compile(r'''Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
a							      =     re.compile(r'''(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')


# This is to make sure the transformers module imported is the one in the repo.
a							      =     direct_transformers_import(TRANSFORMERS_PATH)






def 		_snake_case	(   _snake_case				:   Dict				)       ->   List[str]:






      '''simple docstring'''


      _A     =  re.finditer('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'							,    _snake_case				)
      return [m.group(0				) for m in matches]






def 		_snake_case	(   _snake_case				:   str							,    _snake_case				:   Any				)       ->   int:






      '''simple docstring'''


      _A     =  2 if text == '✅' or text == '❌' else len(_snake_case				)
      _A     =  (width - text_length) // 2
      _A     =  width - text_length - left_indent
      return " " * left_indent + text + " " * right_indent






def 		_snake_case	(   )       ->   Optional[Any]:






      '''simple docstring'''


      _A     =  transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
      _A     =  {
          name: config_maping_names[code]
          for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
          if code in config_maping_names
      }
      _A     =  {name: config.replace('Config'							,    ''				) for name, config in model_name_to_config.items()}

      # Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
      _A     =  collections.defaultdict(_snake_case				)
      _A     =  collections.defaultdict(_snake_case				)
      _A     =  collections.defaultdict(_snake_case				)
      _A     =  collections.defaultdict(_snake_case				)
      _A     =  collections.defaultdict(_snake_case				)

      # Let's lookup through all transformers object (once).
      for attr_name in dir(_snake_case				):
            _A     =  None
            if attr_name.endswith('Tokenizer'				):
                  _A     =  slow_tokenizers
                  _A     =  attr_name[:-9]
            elif attr_name.endswith('TokenizerFast'				):
                  _A     =  fast_tokenizers
                  _A     =  attr_name[:-13]
            elif _re_tf_models.match(_snake_case				) is not None:
                  _A     =  tf_models
                  _A     =  _re_tf_models.match(_snake_case				).groups()[0]
            elif _re_flax_models.match(_snake_case				) is not None:
                  _A     =  flax_models
                  _A     =  _re_flax_models.match(_snake_case				).groups()[0]
            elif _re_pt_models.match(_snake_case				) is not None:
                  _A     =  pt_models
                  _A     =  _re_pt_models.match(_snake_case				).groups()[0]

            if lookup_dict is not None:
                  while len(_snake_case				) > 0:
                        if attr_name in model_name_to_prefix.values():
                              _A     =  True
                              break
                        # Try again after removing the last word in the name
                        _A     =  ''.join(camel_case_split(_snake_case				)[:-1]				)

    # Let's build that table!
      _A     =  list(model_name_to_config.keys()				)
      model_names.sort(key=str.lower				)
      _A     =  ['Model', 'Tokenizer slow', 'Tokenizer fast', 'PyTorch support', 'TensorFlow support', 'Flax Support']
      # We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
      _A     =  [len(_snake_case				) + 2 for c in columns]
      _A     =  max([len(_snake_case				) for name in model_names]				) + 2

      # Build the table per se
      _A     =  '|' + '|'.join([_center_text(_snake_case							,    _snake_case				) for c, w in zip(_snake_case							,    _snake_case				)]				) + '|\n'
      # Use ":-----:" format to center-aligned table cell texts
      table += "|" + "|".join([':' + '-' * (w - 2) + ':' for w in widths]				) + "|\n"

      _A     =  {True: '✅', False: '❌'}
      for name in model_names:
            _A     =  model_name_to_prefix[name]
            _A     =  [
                name,
                check[slow_tokenizers[prefix]],
                check[fast_tokenizers[prefix]],
                check[pt_models[prefix]],
                check[tf_models[prefix]],
                check[flax_models[prefix]],
            ]
            table += "|" + "|".join([_center_text(_snake_case							,    _snake_case				) for l, w in zip(_snake_case							,    _snake_case				)]				) + "|\n"
      return table






def 		_snake_case	(   _snake_case				:   List[str]=False				)       ->   Union[str, Any]:






      '''simple docstring'''


      _A  ,       _A  ,       _A  ,       _A     =  _find_text_in_file(
          filename=os.path.join(_snake_case							,    'index.md'				)							,    start_prompt='<!--This table is updated automatically from the auto modules'							,    end_prompt='<!-- End table-->'							,    )
      _A     =  get_model_table_from_auto_modules()

      if current_table != new_table:
            if overwrite:
                  with open(os.path.join(_snake_case							,    'index.md'				)							,    'w'							,    encoding='utf-8'							,    newline='\n'				) as f:
                        f.writelines(lines[:start_index] + [new_table] + lines[end_index:]				)
            else:
                  raise ValueError(
                      'The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this.'				)


if __name__ == "__main__":
       a							      =     argparse.ArgumentParser()
       parser.add_argument('''--fix_and_overwrite''', action='''store_true''', help='''Whether to fix inconsistencies.''')
       a							      =     parser.parse_args()

       check_model_table(args.fix_and_overwrite)



"""simple docstring"""


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


SCREAMING_SNAKE_CASE			  =  logging.getLogger(__name__)



def 						lowerCamelCase__      (				)->				Tuple:







       """simple docstring"""
       UpperCamelCase       			=		argparse.ArgumentParser(
           description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."					)
       parser.add_argument(
           "--dataset_name"		,   type=UpperCAmelCase_		,   default="wikitext"		,   help="Name of the training. Explore datasets at: hf.co/datasets."		,   )
       parser.add_argument(
           "--dataset_config"		,   type=UpperCAmelCase_		,   default="wikitext-103-raw-v1"		,   help="Configuration name of the dataset."					)
       parser.add_argument(
           "--tokenizer_name_or_path"		,   type=UpperCAmelCase_		,   default="sayakpaul/unigram-tokenizer-wikitext"		,   help="Tokenizer identifier. Can be a local filepath or a Hub identifier."		,   )
       parser.add_argument(
           "--shard_size"		,   type=UpperCAmelCase_		,   default=10_00		,   help="Number of entries to go in a single shard."		,   )
       parser.add_argument("--split"		,   type=UpperCAmelCase_		,   default="train"		,   choices=["train", "test", "validation"]					)
       parser.add_argument(
           "--limit"		,   default=UpperCAmelCase_		,   type=UpperCAmelCase_		,   help="Limit the number of shards (used for debugging)."		,   )
       parser.add_argument(
           "--max_length"		,   type=UpperCAmelCase_		,   default=5_12		,   help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
           " sequence length that is a multiple of 8."		,   )
       parser.add_argument(
           "--output_dir"		,   default="tf-tpu"		,   type=UpperCAmelCase_		,   help="Output directory where the TFRecord shards will be saved. If the"
           " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
           " shards will be directly saved to a Google Cloud Storage bucket."		,   )

       UpperCamelCase       			=		parser.parse_args()
       return args



def 						lowerCamelCase__      (				UpperCAmelCase_					)->				Optional[int]:







       """simple docstring"""
       def fn(UpperCAmelCase_					):
              return tokenizer(examples["text"]					)

       return fn



def 						lowerCamelCase__      (				UpperCAmelCase_					)->				Union[str, Any]:







       """simple docstring"""
       UpperCamelCase       			=		[]
       for i in range(len(tokenized_data["input_ids"]					)					):
              UpperCamelCase       			=		{
                  "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]					)					),
                  "attention_mask": tf.train.Feature(
                      intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]					)					),
              }
              UpperCamelCase       			=		tf.train.Features(feature=UpperCAmelCase_					)
              UpperCamelCase       			=		tf.train.Example(features=UpperCAmelCase_					)
              UpperCamelCase       			=		example.SerializeToString()
              records.append(UpperCAmelCase_					)
       return records



def 						lowerCamelCase__      (				UpperCAmelCase_					)->				int:







       """simple docstring"""
       UpperCamelCase       			=		datasets.load_dataset(args.dataset_name		,   args.dataset_config		,   split=args.split					)

       if args.limit is not None:
              UpperCamelCase       			=		min(len(UpperCAmelCase_					)		,   args.limit					)
              UpperCamelCase       			=		dataset.select(range(UpperCAmelCase_					)					)
              print(F"Limiting the dataset to {args.limit} entries."					)

       UpperCamelCase       			=		AutoTokenizer.from_pretrained(args.tokenizer_name_or_path					)

       # Handle output directory creation.
       # For serializing into a Google Cloud Storage Bucket, one needs to first
       # create a bucket.
       if "gs" not in args.output_dir:
              if not os.path.exists(args.output_dir					):
                     os.makedirs(args.output_dir					)
              UpperCamelCase       			=		os.path.join(args.output_dir		,   args.split					)
              if not os.path.exists(UpperCAmelCase_					):
                     os.makedirs(UpperCAmelCase_					)
       else:
              UpperCamelCase       			=		os.path.join(args.output_dir		,   args.split					)

       # Tokenize the whole dataset at once.
       UpperCamelCase       			=		tokenize_function(UpperCAmelCase_					)
       UpperCamelCase       			=		dataset.map(UpperCAmelCase_		,   batched=UpperCAmelCase_		,   num_proc=4		,   remove_columns=["text"]					)

       # We need to concatenate all our texts together, and then split the result
       # into chunks of a fixed size, which we will call block_size. To do this, we
       # will use the map method again, with the option batched=True. When we use batched=True,
       # the function we pass to map() will be passed multiple inputs at once, allowing us
       # to group them into more or fewer examples than we had in the input.
       # This allows us to create our new fixed-length samples. The advantage of this
       # method is that we don't lose a whole lot of content from the dataset compared to the
       # case where we simply tokenize with a pre-defined max_length.

       def group_texts(UpperCAmelCase_					):
              # Concatenate all texts.
              UpperCamelCase       			=		{k: sum(examples[k]		,   []					) for k in examples.keys()}
              UpperCamelCase       			=		len(concatenated_examples[list(examples.keys()					)[0]]					)
              # We drop the small remainder, though you could add padding instead if the model supports it
              # In this, as in all things, we advise you to follow your heart 🫀
              UpperCamelCase       			=		(total_length // args.max_length) * args.max_length
              # Split by chunks of max_len.
              UpperCamelCase       			=		{
                  k: [t[i : i + args.max_length] for i in range(0		,   UpperCAmelCase_		,   args.max_length					)]
                  for k, t in concatenated_examples.items()
              }
              return result

       UpperCamelCase       			=		dataset_tokenized.map(UpperCAmelCase_		,   batched=UpperCAmelCase_		,   batch_size=10_00		,   num_proc=4					)

       UpperCamelCase       			=		0
       UpperCamelCase       			=		0
       for shard in range(0		,   len(UpperCAmelCase_					)		,   args.shard_size					):
              UpperCamelCase       			=		grouped_dataset[shard : shard + args.shard_size]
              UpperCamelCase       			=		len(dataset_snapshot["input_ids"]					)
              UpperCamelCase       			=		os.path.join(UpperCAmelCase_		,   F"dataset-{shard_count}-{records_containing}.tfrecord"					)
              UpperCamelCase       			=		get_serialized_examples(UpperCAmelCase_					)

              with tf.io.TFRecordWriter(UpperCAmelCase_					) as out_file:
                     for i in range(len(UpperCAmelCase_					)					):
                            UpperCamelCase       			=		serialized_examples[i]
                            out_file.write(UpperCAmelCase_					)
                     print("Wrote file {} containing {} records".format(UpperCAmelCase_		,   UpperCAmelCase_					)					)

              shard_count += 1
              total_records += records_containing

       with open(F"split-{args.split}-records-count.txt"		,   "w"					) as f:
              print(F"Total {args.split} records: {total_records}"		,   file=UpperCAmelCase_					)


if __name__ == "__main__":
      SCREAMING_SNAKE_CASE			  =  parse_args()
      main(args)





