



'''simple docstring'''






import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


lowerCAmelCase:      Union[str, Any]		 =		re.compile(r'\s+')
def    lowerCamelCase__ (		_A       ):
  return {"hash": hashlib.mda(re.sub(_A				,    ''				,    example['content']       ).encode('utf-8'       )       ).hexdigest()}
def    lowerCamelCase__ (		_A       ):
  a		:      Union[str, Any]       =  [len(_A       ) for line in example['content'].splitlines()]
  return {"line_mean": np.mean(_A       ), "line_max": max(_A       )}
def    lowerCamelCase__ (		_A       ):
  a		:      Dict       =  np.mean([c.isalnum() for c in example['content']]       )
  return {"alpha_frac": alpha_frac}
def    lowerCamelCase__ (		_A				,    _A       ):
  if example["hash"] in uniques:
    uniques.remove(example['hash']       )
    return True
  else:
    return False
def    lowerCamelCase__ (		_A				,    _A=5       ):
  a		:      Dict       =  ['auto-generated', 'autogenerated', 'automatically generated']
  a		:      Union[str, Any]       =  example['content'].splitlines()
  for _, line in zip(range(_A       )				,    _A       ):
    for keyword in keywords:
      if keyword in line.lower():
        return {"autogenerated": True}
  else:
    return {"autogenerated": False}
def    lowerCamelCase__ (		_A				,    _A=5				,    _A=0.05       ):
  a		:      Optional[int]       =  ['unit tests', 'test file', 'configuration file']
  a		:      Optional[int]       =  example['content'].splitlines()
  a		:      Any       =  0
  a		:      int       =  0
  # first test
  for _, line in zip(range(_A       )				,    _A       ):
    for keyword in keywords:
      if keyword in line.lower():
        return {"config_or_test": True}
    # second test
  a		:      str       =  example['content'].count('\n'       )
  a		:      List[str]       =  int(coeff * nlines       )
  for line in lines:
    count_config += line.lower().count('config'       )
    count_test += line.lower().count('test'       )
    if count_config > threshold or count_test > threshold:
      return {"config_or_test": True}
  return {"config_or_test": False}
def    lowerCamelCase__ (		_A       ):
  a		:      List[str]       =  ['def ', 'class ', 'for ', 'while ']
  a		:      Union[str, Any]       =  example['content'].splitlines()
  for line in lines:
    for keyword in keywords:
      if keyword in line.lower():
        return {"has_no_keywords": False}
  return {"has_no_keywords": True}
def    lowerCamelCase__ (		_A				,    _A=4       ):
  a		:      str       =  example['content'].splitlines()
  a		:      str       =  0
  for line in lines:
    counter += line.lower().count('='       )
    if counter > minimum:
      return {"has_few_assignments": False}
  return {"has_few_assignments": True}
def    lowerCamelCase__ (		_A       ):
  a		:      Tuple       =  tokenizer(example['content']				,    truncation=_A       )['input_ids']
  a		:      int       =  len(example['content']       ) / len(_A       )
  return {"ratio": ratio}
def    lowerCamelCase__ (		_A       ):
  a		:      Union[str, Any]       =  {}
  results.update(get_hash(_A       )       )
  results.update(line_stats(_A       )       )
  results.update(alpha_stats(_A       )       )
  results.update(char_token_ratio(_A       )       )
  results.update(is_autogenerated(_A       )       )
  results.update(is_config_or_test(_A       )       )
  results.update(has_no_keywords(_A       )       )
  results.update(has_few_assignments(_A       )       )
  return results
def    lowerCamelCase__ (		_A				,    _A				,    _A       ):
  if not check_uniques(_A				,    _A       ):
    return False
  elif example["autogenerated"]:
    return False
  elif example["line_max"] > args.line_max:
    return False
  elif example["line_mean"] > args.line_mean:
    return False
  elif example["alpha_frac"] < args.alpha_frac:
    return False
  elif example["ratio"] < args.min_token_ratio:
    return False
  elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
    return False
  elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
    return False
  elif example["has_few_assignments"]:
    return False
  else:
    return True
def    lowerCamelCase__ (		_A       ):
  with open(_A				,    'rb'       ) as f_in:
    with gzip.open(str(_A       ) + '.gz'				,    'wb'				,    compresslevel=6       ) as f_out:
      shutil.copyfileobj(_A				,    _A       )
  os.unlink(_A       )


# Settings
lowerCAmelCase:      List[str]		 =		HfArgumentParser(PreprocessingArguments)
lowerCAmelCase:      Any		 =		parser.parse_args()
if args.num_workers is None:
     lowerCAmelCase:      List[str]		 =		multiprocessing.cpu_count()
lowerCAmelCase:      List[str]		 =		AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
lowerCAmelCase:      Tuple		 =		time.time()
lowerCAmelCase:      Tuple		 =		load_dataset(args.dataset_name, split='train')
print(F"Time to load dataset: {time.time()-t_start:.2f}")

# Run preprocessing
lowerCAmelCase:      str		 =		time.time()
lowerCAmelCase:      str		 =		ds.map(preprocess, num_proc=args.num_workers)
print(F"Time to preprocess dataset: {time.time()-t_start:.2f}")

# Deduplicate hashes
lowerCAmelCase:      Any		 =		set(ds.unique('hash'))
lowerCAmelCase:      Optional[int]		 =		len(uniques) / len(ds)
print(F"Fraction of duplicates: {1-frac:.2%}")

# Deduplicate data and apply heuristics
lowerCAmelCase:      Dict		 =		time.time()
lowerCAmelCase:      List[Any]		 =		ds.filter(filter, fn_kwargs={'uniques': uniques, 'args': args})
print(F"Time to filter dataset: {time.time()-t_start:.2f}")
print(F"Size of filtered dataset: {len(ds_filter)}")

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
     lowerCAmelCase:      Any		 =		time.time()
     lowerCAmelCase ,	lowerCAmelCase:      Optional[int]		 =		deduplicate_dataset(ds_filter, args.jaccard_threshold)
     print(F"Time to deduplicate dataset: {time.time()-t_start:.2f}")
     print(F"Size of deduplicate dataset: {len(ds_filter)}")

# Save data in batches of samples_per_file
lowerCAmelCase:      Optional[Any]		 =		Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
     with open(output_dir / 'duplicate_clusters.json', 'w') as f:
          json.dump(duplicate_clusters, f)

lowerCAmelCase:      int		 =		output_dir / 'data'
data_dir.mkdir(exist_ok=True)

lowerCAmelCase:      Optional[Any]		 =		time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
     lowerCAmelCase:      Optional[int]		 =		str(data_dir / F"file-{file_number+1:012}.json")
     lowerCAmelCase:      Union[str, Any]		 =		min(len(ds_filter), index + args.samples_per_file)
     ds_filter.select(list(range(index, end_index))).to_json(file_path)
     compress_file(file_path)

print(F"Time to save dataset: {time.time()-t_start:.2f}")




'''simple docstring'''






import argparse
import collections
import os
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
lowerCAmelCase:      Tuple		 =		'src/transformers'
lowerCAmelCase:      Union[str, Any]		 =		'docs/source/en'
lowerCAmelCase:      Dict		 =		'.'
def    lowerCamelCase__ (		_A				,    _A				,    _A       ):
  with open(_A				,    'r'				,    encoding='utf-8'				,    newline='\n'       ) as f:
    a		:      Optional[Any]       =  f.readlines()
  # Find the start prompt.
  a		:      Dict       =  0
  while not lines[start_index].startswith(_A       ):
    start_index += 1
  start_index += 1

  a		:      Optional[Any]       =  start_index
  while not lines[end_index].startswith(_A       ):
    end_index += 1
  end_index -= 1

  while len(lines[start_index]       ) <= 1:
    start_index += 1
  while len(lines[end_index]       ) <= 1:
    end_index -= 1
  end_index += 1
  return "".join(lines[start_index:end_index]       ), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
lowerCAmelCase:      List[str]		 =		'Model|Encoder|Decoder|ForConditionalGeneration'
# Regexes that match TF/Flax/PT model names.
lowerCAmelCase:      Dict		 =		re.compile(r'TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
lowerCAmelCase:      Any		 =		re.compile(r'Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
lowerCAmelCase:      Dict		 =		re.compile(r'(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')


# This is to make sure the transformers module imported is the one in the repo.
lowerCAmelCase:      List[Any]		 =		direct_transformers_import(TRANSFORMERS_PATH)
def    lowerCamelCase__ (		_A       ):
  a		:      Tuple       =  re.finditer('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'				,    _A       )
  return [m.group(0       ) for m in matches]
def    lowerCamelCase__ (		_A				,    _A       ):
  a		:      List[Any]       =  2 if text == '✅' or text == '❌' else len(_A       )
  a		:      Optional[int]       =  (width - text_length) // 2
  a		:      List[Any]       =  width - text_length - left_indent
  return " " * left_indent + text + " " * right_indent
def    lowerCamelCase__ (		):
  a		:      Union[str, Any]       =  transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
  a		:      Union[str, Any]       =  {
      name: config_maping_names[code]
      for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
      if code in config_maping_names
  }
  a		:      Tuple       =  {name: config.replace('Config'				,    ''       ) for name, config in model_name_to_config.items()}

  # Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
  a		:      int       =  collections.defaultdict(_A       )
  a		:      List[Any]       =  collections.defaultdict(_A       )
  a		:      List[Any]       =  collections.defaultdict(_A       )
  a		:      Union[str, Any]       =  collections.defaultdict(_A       )
  a		:      int       =  collections.defaultdict(_A       )

  # Let's lookup through all transformers object (once).
  for attr_name in dir(_A       ):
    a		:      Optional[Any]       =  None
    if attr_name.endswith('Tokenizer'       ):
      a		:      int       =  slow_tokenizers
      a		:      Any       =  attr_name[:-9]
    elif attr_name.endswith('TokenizerFast'       ):
      a		:      str       =  fast_tokenizers
      a		:      List[Any]       =  attr_name[:-13]
    elif _re_tf_models.match(_A       ) is not None:
      a		:      Optional[Any]       =  tf_models
      a		:      Dict       =  _re_tf_models.match(_A       ).groups()[0]
    elif _re_flax_models.match(_A       ) is not None:
      a		:      int       =  flax_models
      a		:      Optional[Any]       =  _re_flax_models.match(_A       ).groups()[0]
    elif _re_pt_models.match(_A       ) is not None:
      a		:      Tuple       =  pt_models
      a		:      Optional[int]       =  _re_pt_models.match(_A       ).groups()[0]

    if lookup_dict is not None:
      while len(_A       ) > 0:
        if attr_name in model_name_to_prefix.values():
          a		:      Tuple       =  True
          break
        # Try again after removing the last word in the name
        a		:      Tuple       =  ''.join(camel_case_split(_A       )[:-1]       )

    # Let's build that table!
  a		:      List[Any]       =  list(model_name_to_config.keys()       )
  model_names.sort(key=str.lower       )
  a		:      Tuple       =  ['Model', 'Tokenizer slow', 'Tokenizer fast', 'PyTorch support', 'TensorFlow support', 'Flax Support']
  # We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
  a		:      Tuple       =  [len(_A       ) + 2 for c in columns]
  a		:      str       =  max([len(_A       ) for name in model_names]       ) + 2

  # Build the table per se
  a		:      List[Any]       =  '|' + '|'.join([_center_text(_A				,    _A       ) for c, w in zip(_A				,    _A       )]       ) + '|\n'
  # Use ":-----:" format to center-aligned table cell texts
  table += "|" + "|".join([':' + '-' * (w - 2) + ':' for w in widths]       ) + "|\n"

  a		:      str       =  {True: '✅', False: '❌'}
  for name in model_names:
    a		:      Any       =  model_name_to_prefix[name]
    a		:      Optional[Any]       =  [
        name,
        check[slow_tokenizers[prefix]],
        check[fast_tokenizers[prefix]],
        check[pt_models[prefix]],
        check[tf_models[prefix]],
        check[flax_models[prefix]],
    ]
    table += "|" + "|".join([_center_text(_A				,    _A       ) for l, w in zip(_A				,    _A       )]       ) + "|\n"
  return table
def    lowerCamelCase__ (		_A=False       ):
  a     ,			a     ,			a     ,			a		:      Tuple       =  _find_text_in_file(
      filename=os.path.join(_A				,    'index.md'       )				,    start_prompt='<!--This table is updated automatically from the auto modules'				,    end_prompt='<!-- End table-->'				,    )
  a		:      Optional[int]       =  get_model_table_from_auto_modules()

  if current_table != new_table:
    if overwrite:
      with open(os.path.join(_A				,    'index.md'       )				,    'w'				,    encoding='utf-8'				,    newline='\n'       ) as f:
        f.writelines(lines[:start_index] + [new_table] + lines[end_index:]       )
    else:
      raise ValueError(
          'The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this.'       )


if __name__ == "__main__":
     lowerCAmelCase:      Dict		 =		argparse.ArgumentParser()
     parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
     lowerCAmelCase:      Any		 =		parser.parse_args()

     check_model_table(args.fix_and_overwrite)