

import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


lowercase_             =     re.compile(R'\s+')
def   a   (    A__			:     str  )					->   Optional[int]:





      """simple docstring"""



      return {"hash": hashlib.mda(re.sub(A__   ,     ''   ,     example['content']  ).encode('utf-8'  )  ).hexdigest()}
def   a   (    A__			:     int  )					->   Any:





      """simple docstring"""



      _lowercase			 =[len(A__  ) for line in example['content'].splitlines()]
      return {"line_mean": np.mean(A__  ), "line_max": max(A__  )}
def   a   (    A__			:     Dict  )					->   int:





      """simple docstring"""



      _lowercase			 =np.mean([c.isalnum() for c in example['content']]  )
      return {"alpha_frac": alpha_frac}
def   a   (    A__			:     Optional[int]   ,     A__			:     Dict  )					->   Union[str, Any]:





      """simple docstring"""



      if example["hash"] in uniques:
            uniques.remove(example['hash']  )
            return True
      else:
            return False
def   a   (    A__			:     List[Any]   ,     A__			:     List[str]=5  )					->   Optional[int]:





      """simple docstring"""



      _lowercase			 =['auto-generated', 'autogenerated', 'automatically generated']
      _lowercase			 =example['content'].splitlines()
      for _, line in zip(range(A__  )   ,     A__  ):
            for keyword in keywords:
                  if keyword in line.lower():
                        return {"autogenerated": True}
      else:
            return {"autogenerated": False}
def   a   (    A__			:     str   ,     A__			:     Any=5   ,     A__			:     Dict=0.05  )					->   List[Any]:





      """simple docstring"""



      _lowercase			 =['unit tests', 'test file', 'configuration file']
      _lowercase			 =example['content'].splitlines()
      _lowercase			 =0
      _lowercase			 =0
      # first test
      for _, line in zip(range(A__  )   ,     A__  ):
            for keyword in keywords:
                  if keyword in line.lower():
                        return {"config_or_test": True}
    # second test
      _lowercase			 =example['content'].count('\n'  )
      _lowercase			 =int(coeff * nlines  )
      for line in lines:
            count_config += line.lower().count('config'  )
            count_test += line.lower().count('test'  )
            if count_config > threshold or count_test > threshold:
                  return {"config_or_test": True}
      return {"config_or_test": False}
def   a   (    A__			:     List[Any]  )					->   Dict:





      """simple docstring"""



      _lowercase			 =['def ', 'class ', 'for ', 'while ']
      _lowercase			 =example['content'].splitlines()
      for line in lines:
            for keyword in keywords:
                  if keyword in line.lower():
                        return {"has_no_keywords": False}
      return {"has_no_keywords": True}
def   a   (    A__			:     Tuple   ,     A__			:     str=4  )					->   List[Any]:





      """simple docstring"""



      _lowercase			 =example['content'].splitlines()
      _lowercase			 =0
      for line in lines:
            counter += line.lower().count('='  )
            if counter > minimum:
                  return {"has_few_assignments": False}
      return {"has_few_assignments": True}
def   a   (    A__			:     Optional[int]  )					->   Union[str, Any]:





      """simple docstring"""



      _lowercase			 =tokenizer(example['content']   ,     truncation=A__  )['input_ids']
      _lowercase			 =len(example['content']  ) / len(A__  )
      return {"ratio": ratio}
def   a   (    A__			:     List[Any]  )					->   Optional[int]:





      """simple docstring"""



      _lowercase			 ={}
      results.update(get_hash(A__  )  )
      results.update(line_stats(A__  )  )
      results.update(alpha_stats(A__  )  )
      results.update(char_token_ratio(A__  )  )
      results.update(is_autogenerated(A__  )  )
      results.update(is_config_or_test(A__  )  )
      results.update(has_no_keywords(A__  )  )
      results.update(has_few_assignments(A__  )  )
      return results
def   a   (    A__			:     Tuple   ,     A__			:     Union[str, Any]   ,     A__			:     List[Any]  )					->   List[Any]:





      """simple docstring"""



      if not check_uniques(A__   ,     A__  ):
            return False
      elif example["autogenerated"]:
            return False
      elif example["line_max"] > args.line_max:
            return False
      elif example["line_mean"] > args.line_mean:
            return False
      elif example["alpha_frac"] < args.alpha_frac:
            return False
      elif example["ratio"] < args.min_token_ratio:
            return False
      elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
            return False
      elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
            return False
      elif example["has_few_assignments"]:
            return False
      else:
            return True
def   a   (    A__			:     int  )					->   Any:





      """simple docstring"""



      with open(A__   ,     'rb'  ) as f_in:
            with gzip.open(str(A__  ) + '.gz'   ,     'wb'   ,     compresslevel=6  ) as f_out:
                  shutil.copyfileobj(A__   ,     A__  )
      os.unlink(A__  )


# Settings
lowercase_             =     HfArgumentParser(PreprocessingArguments)
lowercase_             =     parser.parse_args()
if args.num_workers is None:
  lowercase_             =     multiprocessing.cpu_count()
lowercase_             =     AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
lowercase_             =     time.time()
lowercase_             =     load_dataset(args.dataset_name, split='train')
print(f"Time to load dataset: {time.time()-t_start:.2f}")

# Run preprocessing
lowercase_             =     time.time()
lowercase_             =     ds.map(preprocess, num_proc=args.num_workers)
print(f"Time to preprocess dataset: {time.time()-t_start:.2f}")

# Deduplicate hashes
lowercase_             =     set(ds.unique('hash'))
lowercase_             =     len(uniques) / len(ds)
print(f"Fraction of duplicates: {1-frac:.2%}")

# Deduplicate data and apply heuristics
lowercase_             =     time.time()
lowercase_             =     ds.filter(filter, fn_kwargs={'uniques': uniques, 'args': args})
print(f"Time to filter dataset: {time.time()-t_start:.2f}")
print(f"Size of filtered dataset: {len(ds_filter)}")

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
  lowercase_             =     time.time()
  lowercase_       ,		lowercase_             =     deduplicate_dataset(ds_filter, args.jaccard_threshold)
  print(f"Time to deduplicate dataset: {time.time()-t_start:.2f}")
  print(f"Size of deduplicate dataset: {len(ds_filter)}")

# Save data in batches of samples_per_file
lowercase_             =     Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
  with open(output_dir / 'duplicate_clusters.json', 'w') as f:
    json.dump(duplicate_clusters, f)

lowercase_             =     output_dir / 'data'
data_dir.mkdir(exist_ok=True)

lowercase_             =     time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
  lowercase_             =     str(data_dir / f"file-{file_number+1:012}.json")
  lowercase_             =     min(len(ds_filter), index + args.samples_per_file)
  ds_filter.select(list(range(index, end_index))).to_json(file_path)
  compress_file(file_path)







print(f"Time to save dataset: {time.time()-t_start:.2f}")





import argparse
import os

from . import (
    ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    BART_PRETRAINED_MODEL_ARCHIVE_LIST,
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP,
    DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
    DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
    DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
    ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP,
    FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
    LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP,
    T5_PRETRAINED_CONFIG_ARCHIVE_MAP,
    TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP,
    WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    XLM_PRETRAINED_CONFIG_ARCHIVE_MAP,
    XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP,
    XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AlbertConfig,
    BartConfig,
    BertConfig,
    CamembertConfig,
    CTRLConfig,
    DistilBertConfig,
    DPRConfig,
    ElectraConfig,
    FlaubertConfig,
    GPTaConfig,
    LayoutLMConfig,
    LxmertConfig,
    OpenAIGPTConfig,
    RobertaConfig,
    TaConfig,
    TFAlbertForPreTraining,
    TFBartForConditionalGeneration,
    TFBartForSequenceClassification,
    TFBertForPreTraining,
    TFBertForQuestionAnswering,
    TFBertForSequenceClassification,
    TFCamembertForMaskedLM,
    TFCTRLLMHeadModel,
    TFDistilBertForMaskedLM,
    TFDistilBertForQuestionAnswering,
    TFDPRContextEncoder,
    TFDPRQuestionEncoder,
    TFDPRReader,
    TFElectraForPreTraining,
    TFFlaubertWithLMHeadModel,
    TFGPTaLMHeadModel,
    TFLayoutLMForMaskedLM,
    TFLxmertForPreTraining,
    TFLxmertVisualFeatureEncoder,
    TFOpenAIGPTLMHeadModel,
    TFRobertaForCausalLM,
    TFRobertaForMaskedLM,
    TFRobertaForSequenceClassification,
    TFTaForConditionalGeneration,
    TFTransfoXLLMHeadModel,
    TFWavaVecaModel,
    TFXLMRobertaForMaskedLM,
    TFXLMWithLMHeadModel,
    TFXLNetLMHeadModel,
    TransfoXLConfig,
    WavaVecaConfig,
    WavaVecaModel,
    XLMConfig,
    XLMRobertaConfig,
    XLNetConfig,
    is_torch_available,
    load_pytorch_checkpoint_in_tfa_model,
)
from .utils import CONFIG_NAME, WEIGHTS_NAME, cached_file, logging


if is_torch_available():
  import numpy as np
  import torch

  from . import (
      AlbertForPreTraining,
      BartForConditionalGeneration,
      BertForPreTraining,
      BertForQuestionAnswering,
      BertForSequenceClassification,
      CamembertForMaskedLM,
      CTRLLMHeadModel,
      DistilBertForMaskedLM,
      DistilBertForQuestionAnswering,
      DPRContextEncoder,
      DPRQuestionEncoder,
      DPRReader,
      ElectraForPreTraining,
      FlaubertWithLMHeadModel,
      GPTaLMHeadModel,
      LayoutLMForMaskedLM,
      LxmertForPreTraining,
      LxmertVisualFeatureEncoder,
      OpenAIGPTLMHeadModel,
      RobertaForMaskedLM,
      RobertaForSequenceClassification,
      TaForConditionalGeneration,
      TransfoXLLMHeadModel,
      XLMRobertaForMaskedLM,
      XLMWithLMHeadModel,
      XLNetLMHeadModel,
  )


logging.set_verbosity_info()

lowercase_             =     {
    'bart': (
        BartConfig,
        TFBartForConditionalGeneration,
        TFBartForSequenceClassification,
        BartForConditionalGeneration,
        BART_PRETRAINED_MODEL_ARCHIVE_LIST,
    ),
    'bert': (
        BertConfig,
        TFBertForPreTraining,
        BertForPreTraining,
        BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'bert-large-uncased-whole-word-masking-finetuned-squad': (
        BertConfig,
        TFBertForQuestionAnswering,
        BertForQuestionAnswering,
        BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'bert-large-cased-whole-word-masking-finetuned-squad': (
        BertConfig,
        TFBertForQuestionAnswering,
        BertForQuestionAnswering,
        BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'bert-base-cased-finetuned-mrpc': (
        BertConfig,
        TFBertForSequenceClassification,
        BertForSequenceClassification,
        BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'dpr': (
        DPRConfig,
        TFDPRQuestionEncoder,
        TFDPRContextEncoder,
        TFDPRReader,
        DPRQuestionEncoder,
        DPRContextEncoder,
        DPRReader,
        DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
        DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
        DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
    ),
    'gpt2': (
        GPTaConfig,
        TFGPTaLMHeadModel,
        GPTaLMHeadModel,
        GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'xlnet': (
        XLNetConfig,
        TFXLNetLMHeadModel,
        XLNetLMHeadModel,
        XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'xlm': (
        XLMConfig,
        TFXLMWithLMHeadModel,
        XLMWithLMHeadModel,
        XLM_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'xlm-roberta': (
        XLMRobertaConfig,
        TFXLMRobertaForMaskedLM,
        XLMRobertaForMaskedLM,
        XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'transfo-xl': (
        TransfoXLConfig,
        TFTransfoXLLMHeadModel,
        TransfoXLLMHeadModel,
        TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'openai-gpt': (
        OpenAIGPTConfig,
        TFOpenAIGPTLMHeadModel,
        OpenAIGPTLMHeadModel,
        OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'roberta': (
        RobertaConfig,
        TFRobertaForCausalLM,
        TFRobertaForMaskedLM,
        RobertaForMaskedLM,
        ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'layoutlm': (
        LayoutLMConfig,
        TFLayoutLMForMaskedLM,
        LayoutLMForMaskedLM,
        LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
    ),
    'roberta-large-mnli': (
        RobertaConfig,
        TFRobertaForSequenceClassification,
        RobertaForSequenceClassification,
        ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'camembert': (
        CamembertConfig,
        TFCamembertForMaskedLM,
        CamembertForMaskedLM,
        CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'flaubert': (
        FlaubertConfig,
        TFFlaubertWithLMHeadModel,
        FlaubertWithLMHeadModel,
        FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'distilbert': (
        DistilBertConfig,
        TFDistilBertForMaskedLM,
        DistilBertForMaskedLM,
        DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'distilbert-base-distilled-squad': (
        DistilBertConfig,
        TFDistilBertForQuestionAnswering,
        DistilBertForQuestionAnswering,
        DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'lxmert': (
        LxmertConfig,
        TFLxmertForPreTraining,
        LxmertForPreTraining,
        LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'lxmert-visual-feature-encoder': (
        LxmertConfig,
        TFLxmertVisualFeatureEncoder,
        LxmertVisualFeatureEncoder,
        LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'ctrl': (
        CTRLConfig,
        TFCTRLLMHeadModel,
        CTRLLMHeadModel,
        CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'albert': (
        AlbertConfig,
        TFAlbertForPreTraining,
        AlbertForPreTraining,
        ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    't5': (
        TaConfig,
        TFTaForConditionalGeneration,
        TaForConditionalGeneration,
        T5_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'electra': (
        ElectraConfig,
        TFElectraForPreTraining,
        ElectraForPreTraining,
        ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
    'wav2vec2': (
        WavaVecaConfig,
        TFWavaVecaModel,
        WavaVecaModel,
        WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ),
}
def   a   (    A__			:     Tuple   ,     A__			:     List[Any]   ,     A__			:     Optional[int]   ,     A__			:     Dict   ,     A__			:     Any=False   ,     A__			:     str=True  )					->   str:





      """simple docstring"""



      if model_type not in MODEL_CLASSES:
            raise ValueError(F'''Unrecognized model type, should be one of {list(MODEL_CLASSES.keys()  )}.'''  )

      _lowercase , _lowercase , _lowercase , _lowercase			 =MODEL_CLASSES[model_type]

      # Initialise TF model
      if config_file in aws_config_map:
            _lowercase			 =cached_file(A__   ,     A__   ,     force_download=not use_cached_models  )
      _lowercase			 =config_class.from_json_file(A__  )
      _lowercase			 =True
      _lowercase			 =True
      print(F'''Building TensorFlow model from configuration: {config}'''  )
      _lowercase			 =model_class(A__  )

      # Load weights from tf checkpoint
      if pytorch_checkpoint_path in aws_config_map.keys():
            _lowercase			 =cached_file(
                A__   ,     A__   ,     force_download=not use_cached_models  )
      # Load PyTorch checkpoint in tf2 model:
      _lowercase			 =load_pytorch_checkpoint_in_tfa_model(A__   ,     A__  )

      if compare_with_pt_model:
            _lowercase			 =tf_model(tf_model.dummy_inputs   ,     training=A__  )  # build the network

            _lowercase			 =torch.load(A__   ,     map_location='cpu'  )
            _lowercase			 =pt_model_class.from_pretrained(
                pretrained_model_name_or_path=A__   ,     config=A__   ,     state_dict=A__  )

            with torch.no_grad():
                  _lowercase			 =pt_model(**pt_model.dummy_inputs  )

            _lowercase			 =pto[0].numpy()
            _lowercase			 =tfo[0].numpy()
            _lowercase			 =np.amax(np.abs(np_pt - np_tf  )  )
            print(F'''Max absolute difference between models outputs {diff}'''  )
            assert diff <= 2e-2, F'''Error, model absolute difference is >2e-2: {diff}'''

      # Save pytorch-model
      print(F'''Save TensorFlow model to {tf_dump_path}'''  )
      tf_model.save_weights(A__   ,     save_format='h5'  )
def   a   (    A__			:     str   ,     A__			:     str   ,     A__			:     Optional[Any]=None   ,     A__			:     Any=None   ,     A__			:     Optional[int]=False   ,     A__			:     Optional[int]=False   ,     A__			:     int=False   ,     A__			:     str=False   ,     )					->   List[Any]:





      """simple docstring"""



      if args_model_type is None:
            _lowercase			 =list(MODEL_CLASSES.keys()  )
      else:
            _lowercase			 =[args_model_type]

      for j, model_type in enumerate(A__   ,     start=1  ):
            print('=' * 100  )
            print(F''' Converting model type {j}/{len(A__  )}: {model_type}'''  )
            print('=' * 100  )
            if model_type not in MODEL_CLASSES:
                  raise ValueError(F'''Unrecognized model type {model_type}, should be one of {list(MODEL_CLASSES.keys()  )}.'''  )

            _lowercase , _lowercase , _lowercase , _lowercase , _lowercase			 =MODEL_CLASSES[model_type]

            if model_shortcut_names_or_path is None:
                  _lowercase			 =list(aws_model_maps.keys()  )
            if config_shortcut_names_or_path is None:
                  _lowercase			 =model_shortcut_names_or_path

            for i, (model_shortcut_name, config_shortcut_name) in enumerate(
                zip(A__   ,     A__  )   ,     start=1  ):
                  print('-' * 100  )
                  if "-squad" in model_shortcut_name or "-mrpc" in model_shortcut_name or "-mnli" in model_shortcut_name:
                        if not only_convert_finetuned_models:
                              print(F'''    Skipping finetuned checkpoint {model_shortcut_name}'''  )
                              continue
                        _lowercase			 =model_shortcut_name
                  elif only_convert_finetuned_models:
                        print(F'''    Skipping not finetuned checkpoint {model_shortcut_name}'''  )
                        continue
                  print(
                      F'''    Converting checkpoint {i}/{len(A__  )}: {model_shortcut_name} - model_type {model_type}'''  )
                  print('-' * 100  )

                  if config_shortcut_name in aws_config_map:
                        _lowercase			 =cached_file(A__   ,     A__   ,     force_download=not use_cached_models  )
                  else:
                        _lowercase			 =config_shortcut_name

                  if model_shortcut_name in aws_model_maps:
                        _lowercase			 =cached_file(A__   ,     A__   ,     force_download=not use_cached_models  )
                  else:
                        _lowercase			 =model_shortcut_name

                  if os.path.isfile(A__  ):
                        _lowercase			 ='converted_model'

                  convert_pt_checkpoint_to_tf(
                      model_type=A__   ,     pytorch_checkpoint_path=A__   ,     config_file=A__   ,     tf_dump_path=os.path.join(A__   ,     model_shortcut_name + '-tf_model.h5'  )   ,     compare_with_pt_model=A__   ,     )
                  if remove_cached_files:
                        os.remove(A__  )
                        os.remove(A__  )


if __name__ == "__main__":
  lowercase_             =     argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      '--tf_dump_path', default=None, type=str, required=True, help='Path to the output Tensorflow dump file.'
  )
  parser.add_argument(
      '--model_type',
      default=None,
      type=str,
      help=(
          f"Model type selected in the list of {list(MODEL_CLASSES.keys())}. If not given, will download and "
          'convert all the models from AWS.'
      ),
  )
  parser.add_argument(
      '--pytorch_checkpoint_path',
      default=None,
      type=str,
      help=(
          'Path to the PyTorch checkpoint path or shortcut name to download from AWS. '
          'If not given, will download and convert all the checkpoints from AWS.'
      ),
  )
  parser.add_argument(
      '--config_file',
      default=None,
      type=str,
      help=(
          'The config json file corresponding to the pre-trained model. \n'
          'This specifies the model architecture. If not given and '
          '--pytorch_checkpoint_path is not given or is a shortcut name '
          'use the configuration associated to the shortcut name on the AWS'
      ),
  )
  parser.add_argument(
      '--compare_with_pt_model', action='store_true', help='Compare Tensorflow and PyTorch model predictions.'
  )
  parser.add_argument(
      '--use_cached_models',
      action='store_true',
      help='Use cached models if possible instead of updating to latest checkpoint versions.',
  )
  parser.add_argument(
      '--remove_cached_files',
      action='store_true',
      help='Remove pytorch models after conversion (save memory when converting in batches).',
  )
  parser.add_argument('--only_convert_finetuned_models', action='store_true', help='Only convert finetuned models.')
  lowercase_             =     parser.parse_args()

  # if args.pytorch_checkpoint_path is not None:
  #     convert_pt_checkpoint_to_tf(args.model_type.lower(),
  #                                 args.pytorch_checkpoint_path,
  #                                 args.config_file if args.config_file is not None else args.pytorch_checkpoint_path,
  #                                 args.tf_dump_path,
  #                                 compare_with_pt_model=args.compare_with_pt_model,
  #                                 use_cached_models=args.use_cached_models)
  # else:
  convert_all_pt_checkpoints_to_tf(
      args.model_type.lower() if args.model_type is not None else None,
      args.tf_dump_path,
      model_shortcut_names_or_path=[args.pytorch_checkpoint_path]
      if args.pytorch_checkpoint_path is not None
      else None,
      config_shortcut_names_or_path=[args.config_file] if args.config_file is not None else None,
      compare_with_pt_model=args.compare_with_pt_model,
      use_cached_models=args.use_cached_models,
      remove_cached_files=args.remove_cached_files,
      only_convert_finetuned_models=args.only_convert_finetuned_models,
  )


