





'''simple docstring'''






import collections
import importlib.util
import os
import re
from pathlib import Path


__lowerCamelCase						: Optional[int]		=   'src/transformers'


# Matches is_xxx_available()
__lowerCamelCase						: str		=   re.compile(R'is\_([a-z_]*)_available()')
# Catches a one-line _import_struct = {xxx}
__lowerCamelCase						: Union[str, Any]		=   re.compile(R'^_import_structure\s+=\s+\{([^\}]+)\}')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
__lowerCamelCase						: str		=   re.compile(R'\s+"\S*":\s+\[([^\]]*)\]')
# Catches a line if not is_foo_available
__lowerCamelCase						: Union[str, Any]		=   re.compile(R'^\s*if\s+not\s+is\_[a-z_]*\_available\(\)')
# Catches a line _import_struct["bla"].append("foo")
__lowerCamelCase						: int		=   re.compile(R'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
__lowerCamelCase						: Tuple		=   re.compile(R'^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]')
# Catches a line with an object between quotes and a comma:     "MyModel",
__lowerCamelCase						: Union[str, Any]		=   re.compile('^\s+"([^"]+)",')
# Catches a line with objects between brackets only:    ["foo", "bar"],
__lowerCamelCase						: Tuple		=   re.compile('^\s+\[([^\]]+)\]')
# Catches a line with from foo import bar, bla, boo
__lowerCamelCase						: int		=   re.compile(R'\s+from\s+\S*\s+import\s+([^\(\s].*)\n')
# Catches a line with try:
__lowerCamelCase						: List[str]		=   re.compile(R'^\s*try:')
# Catches a line with else:
__lowerCamelCase						: Union[str, Any]		=   re.compile(R'^\s*else:')


def 					_a   (__SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  if _re_test_backend.search(__SCREAMING_SNAKE_CASE   ) is None:
    return None
  _UpperCamelCase       			=[b[0] for b in _re_backend.findall(__SCREAMING_SNAKE_CASE   )]
  backends.sort()
  return "_and_".join(__SCREAMING_SNAKE_CASE   )


def 					_a   (__SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  with open(__SCREAMING_SNAKE_CASE  ,      '''r'''  ,      encoding='''utf-8'''  ,      newline='''\n'''   ) as f:
    _UpperCamelCase       			=f.readlines()

  _UpperCamelCase       			=0
  while line_index < len(__SCREAMING_SNAKE_CASE   ) and not lines[line_index].startswith('''_import_structure = {'''   ):
    line_index += 1

  # If this is a traditional init, just return.
  if line_index >= len(__SCREAMING_SNAKE_CASE   ):
    return None

  # First grab the objects without a specific backend in _import_structure
  _UpperCamelCase       			=[]
  while not lines[line_index].startswith('''if TYPE_CHECKING'''   ) and find_backend(lines[line_index]   ) is None:
    _UpperCamelCase       			=lines[line_index]
    # If we have everything on a single line, let's deal with it.
    if _re_one_line_import_struct.search(__SCREAMING_SNAKE_CASE   ):
      _UpperCamelCase       			=_re_one_line_import_struct.search(__SCREAMING_SNAKE_CASE   ).groups()[0]
      _UpperCamelCase       			=re.findall('''\[([^\]]+)\]'''  ,      __SCREAMING_SNAKE_CASE   )
      for imp in imports:
        objects.extend([obj[1:-1] for obj in imp.split(''', '''   )]   )
      line_index += 1
      continue
    _UpperCamelCase       			=_re_import_struct_key_value.search(__SCREAMING_SNAKE_CASE   )
    if single_line_import_search is not None:
      _UpperCamelCase       			=[obj[1:-1] for obj in single_line_import_search.groups()[0].split(''', '''   ) if len(__SCREAMING_SNAKE_CASE   ) > 0]
      objects.extend(__SCREAMING_SNAKE_CASE   )
    elif line.startswith(''' ''' * 8 + '''"'''   ):
      objects.append(line[9:-3]   )
    line_index += 1

  _UpperCamelCase       			={'''none''': objects}
  # Let's continue with backend-specific objects in _import_structure
  while not lines[line_index].startswith('''if TYPE_CHECKING'''   ):
    # If the line is an if not is_backend_available, we grab all objects associated.
    _UpperCamelCase       			=find_backend(lines[line_index]   )
    # Check if the backend declaration is inside a try block:
    if _re_try.search(lines[line_index - 1]   ) is None:
      _UpperCamelCase       			=None

    if backend is not None:
      line_index += 1

      # Scroll until we hit the else block of try-except-else
      while _re_else.search(lines[line_index]   ) is None:
        line_index += 1

      line_index += 1

      _UpperCamelCase       			=[]
      # Until we unindent, add backend objects to the list
      while len(lines[line_index]   ) <= 1 or lines[line_index].startswith(''' ''' * 4   ):
        _UpperCamelCase       			=lines[line_index]
        if _re_import_struct_add_one.search(__SCREAMING_SNAKE_CASE   ) is not None:
          objects.append(_re_import_struct_add_one.search(__SCREAMING_SNAKE_CASE   ).groups()[0]   )
        elif _re_import_struct_add_many.search(__SCREAMING_SNAKE_CASE   ) is not None:
          _UpperCamelCase       			=_re_import_struct_add_many.search(__SCREAMING_SNAKE_CASE   ).groups()[0].split(''', '''   )
          _UpperCamelCase       			=[obj[1:-1] for obj in imports if len(__SCREAMING_SNAKE_CASE   ) > 0]
          objects.extend(__SCREAMING_SNAKE_CASE   )
        elif _re_between_brackets.search(__SCREAMING_SNAKE_CASE   ) is not None:
          _UpperCamelCase       			=_re_between_brackets.search(__SCREAMING_SNAKE_CASE   ).groups()[0].split(''', '''   )
          _UpperCamelCase       			=[obj[1:-1] for obj in imports if len(__SCREAMING_SNAKE_CASE   ) > 0]
          objects.extend(__SCREAMING_SNAKE_CASE   )
        elif _re_quote_object.search(__SCREAMING_SNAKE_CASE   ) is not None:
          objects.append(_re_quote_object.search(__SCREAMING_SNAKE_CASE   ).groups()[0]   )
        elif line.startswith(''' ''' * 8 + '''"'''   ):
          objects.append(line[9:-3]   )
        elif line.startswith(''' ''' * 12 + '''"'''   ):
          objects.append(line[13:-3]   )
        line_index += 1

      _UpperCamelCase       			=objects
    else:
      line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
  _UpperCamelCase       			=[]
  while (
      line_index < len(__SCREAMING_SNAKE_CASE   )
      and find_backend(lines[line_index]   ) is None
      and not lines[line_index].startswith('''else'''   )
  ):
    _UpperCamelCase       			=lines[line_index]
    _UpperCamelCase       			=_re_import.search(__SCREAMING_SNAKE_CASE   )
    if single_line_import_search is not None:
      objects.extend(single_line_import_search.groups()[0].split(''', '''   )   )
    elif line.startswith(''' ''' * 8   ):
      objects.append(line[8:-2]   )
    line_index += 1

  _UpperCamelCase       			={'''none''': objects}
  # Let's continue with backend-specific objects
  while line_index < len(__SCREAMING_SNAKE_CASE   ):
    # If the line is an if is_backend_available, we grab all objects associated.
    _UpperCamelCase       			=find_backend(lines[line_index]   )
    # Check if the backend declaration is inside a try block:
    if _re_try.search(lines[line_index - 1]   ) is None:
      _UpperCamelCase       			=None

    if backend is not None:
      line_index += 1

      # Scroll until we hit the else block of try-except-else
      while _re_else.search(lines[line_index]   ) is None:
        line_index += 1

      line_index += 1

      _UpperCamelCase       			=[]
      # Until we unindent, add backend objects to the list
      while len(lines[line_index]   ) <= 1 or lines[line_index].startswith(''' ''' * 8   ):
        _UpperCamelCase       			=lines[line_index]
        _UpperCamelCase       			=_re_import.search(__SCREAMING_SNAKE_CASE   )
        if single_line_import_search is not None:
          objects.extend(single_line_import_search.groups()[0].split(''', '''   )   )
        elif line.startswith(''' ''' * 12   ):
          objects.append(line[12:-2]   )
        line_index += 1

      _UpperCamelCase       			=objects
    else:
      line_index += 1

  return import_dict_objects, type_hint_objects


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  def find_duplicates(__SCREAMING_SNAKE_CASE   ):
    return [k for k, v in collections.Counter(__SCREAMING_SNAKE_CASE   ).items() if v > 1]

  if list(import_dict_objects.keys()   ) != list(type_hint_objects.keys()   ):
    return ["Both sides of the init do not have the same backends!"]

  _UpperCamelCase       			=[]
  for key in import_dict_objects.keys():
    _UpperCamelCase       			=find_duplicates(import_dict_objects[key]   )
    if duplicate_imports:
      errors.append(f'''Duplicate _import_structure definitions for: {duplicate_imports}'''   )
    _UpperCamelCase       			=find_duplicates(type_hint_objects[key]   )
    if duplicate_type_hints:
      errors.append(f'''Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'''   )

    if sorted(set(import_dict_objects[key]   )   ) != sorted(set(type_hint_objects[key]   )   ):
      _UpperCamelCase       			='''base imports''' if key == '''none''' else f'''{key} backend'''
      errors.append(f'''Differences for {name}:'''   )
      for a in type_hint_objects[key]:
        if a not in import_dict_objects[key]:
          errors.append(f'''  {a} in TYPE_HINT but not in _import_structure.'''   )
      for a in import_dict_objects[key]:
        if a not in type_hint_objects[key]:
          errors.append(f'''  {a} in _import_structure but not in TYPE_HINT.'''   )
  return errors


def 					_a   ():


  """simple docstring"""

  _UpperCamelCase       			=[]
  for root, _, files in os.walk(__SCREAMING_SNAKE_CASE   ):
    if "__init__.py" in files:
      _UpperCamelCase       			=os.path.join(__SCREAMING_SNAKE_CASE  ,      '''__init__.py'''   )
      _UpperCamelCase       			=parse_init(__SCREAMING_SNAKE_CASE   )
      if objects is not None:
        _UpperCamelCase       			=analyze_results(*__SCREAMING_SNAKE_CASE   )
        if len(__SCREAMING_SNAKE_CASE   ) > 0:
          _UpperCamelCase       			=f'''Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'''
          failures.append('''\n'''.join(__SCREAMING_SNAKE_CASE   )   )
  if len(__SCREAMING_SNAKE_CASE   ) > 0:
    raise ValueError('''\n\n'''.join(__SCREAMING_SNAKE_CASE   )   )


def 					_a   ():


  """simple docstring"""

  _UpperCamelCase       			=[]
  for path, directories, files in os.walk(__SCREAMING_SNAKE_CASE   ):
    for folder in directories:
      # Ignore private modules
      if folder.startswith('''_'''   ):
        directories.remove(__SCREAMING_SNAKE_CASE   )
        continue
      # Ignore leftovers from branches (empty folders apart from pycache)
      if len(list((Path(__SCREAMING_SNAKE_CASE   ) / folder).glob('''*.py'''   )   )   ) == 0:
        continue
      _UpperCamelCase       			=str((Path(__SCREAMING_SNAKE_CASE   ) / folder).relative_to(__SCREAMING_SNAKE_CASE   )   )
      _UpperCamelCase       			=short_path.replace(os.path.sep  ,      '''.'''   )
      submodules.append(__SCREAMING_SNAKE_CASE   )
    for fname in files:
      if fname == "__init__.py":
        continue
      _UpperCamelCase       			=str((Path(__SCREAMING_SNAKE_CASE   ) / fname).relative_to(__SCREAMING_SNAKE_CASE   )   )
      _UpperCamelCase       			=short_path.replace('''.py'''  ,      ''''''   ).replace(os.path.sep  ,      '''.'''   )
      if len(submodule.split('''.'''   )   ) == 1:
        submodules.append(__SCREAMING_SNAKE_CASE   )
  return submodules


__lowerCamelCase						: str		=   [
    'convert_pytorch_checkpoint_to_tf2',
    'modeling_flax_pytorch_utils',
]


def 					_a   ():


  """simple docstring"""

  _UpperCamelCase       			=importlib.util.spec_from_file_location(
      '''transformers'''  ,      os.path.join(__SCREAMING_SNAKE_CASE  ,      '''__init__.py'''   )  ,      submodule_search_locations=[PATH_TO_TRANSFORMERS]  ,      )
  _UpperCamelCase       			=spec.loader.load_module()

  _UpperCamelCase       			=[
      module
      for module in get_transformers_submodules()
      if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
  ]
  if len(__SCREAMING_SNAKE_CASE   ) > 0:
    _UpperCamelCase       			='''\n'''.join(f'''- {module}''' for module in module_not_registered   )
    raise ValueError(
        '''The following submodules are not properly registered in the main init of Transformers:\n'''
        f'''{list_of_modules}\n'''
        '''Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'''   )


if __name__ == "__main__":
      check_all_inits()
      check_submodules()













'''simple docstring'''






import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__lowerCamelCase						: List[Any]		=   data_utils.TransfoXLTokenizer
__lowerCamelCase						: str		=   data_utils.TransfoXLCorpus
__lowerCamelCase						: Optional[int]		=   data_utils
__lowerCamelCase						: int		=   data_utils


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  if transfo_xl_dataset_file:
    # Convert a pre-processed corpus (see original TensorFlow repo)
    with open(__SCREAMING_SNAKE_CASE  ,      '''rb'''   ) as fp:
      _UpperCamelCase       			=pickle.load(__SCREAMING_SNAKE_CASE  ,      encoding='''latin1'''   )
    # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
    _UpperCamelCase       			=pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
    print(f'''Save vocabulary to {pytorch_vocab_dump_path}'''   )
    _UpperCamelCase       			=corpus.vocab.__dict__
    torch.save(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )

    _UpperCamelCase       			=corpus.__dict__
    corpus_dict_no_vocab.pop('''vocab'''  ,      __SCREAMING_SNAKE_CASE   )
    _UpperCamelCase       			=pytorch_dump_folder_path + '''/''' + CORPUS_NAME
    print(f'''Save dataset to {pytorch_dataset_dump_path}'''   )
    torch.save(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )

  if tf_checkpoint_path:
    # Convert a pre-trained TensorFlow model
    _UpperCamelCase       			=os.path.abspath(__SCREAMING_SNAKE_CASE   )
    _UpperCamelCase       			=os.path.abspath(__SCREAMING_SNAKE_CASE   )

    print(f'''Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.'''   )
    # Initialise PyTorch model
    if transfo_xl_config_file == "":
      _UpperCamelCase       			=TransfoXLConfig()
    else:
      _UpperCamelCase       			=TransfoXLConfig.from_json_file(__SCREAMING_SNAKE_CASE   )
    print(f'''Building PyTorch model from configuration: {config}'''   )
    _UpperCamelCase       			=TransfoXLLMHeadModel(__SCREAMING_SNAKE_CASE   )

    _UpperCamelCase       			=load_tf_weights_in_transfo_xl(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )
    # Save pytorch-model
    _UpperCamelCase       			=os.path.join(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )
    _UpperCamelCase       			=os.path.join(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )
    print(f'''Save PyTorch model to {os.path.abspath(__SCREAMING_SNAKE_CASE   )}'''   )
    torch.save(model.state_dict()  ,      __SCREAMING_SNAKE_CASE   )
    print(f'''Save configuration file to {os.path.abspath(__SCREAMING_SNAKE_CASE   )}'''   )
    with open(__SCREAMING_SNAKE_CASE  ,      '''w'''  ,      encoding='''utf-8'''   ) as f:
      f.write(config.to_json_string()   )


if __name__ == "__main__":
      __lowerCamelCase						: Any		=   argparse.ArgumentParser()
      parser.add_argument(
          '--pytorch_dump_folder_path',
          default=None,
          type=str,
          required=True,
          help='Path to the folder to store the PyTorch model or dataset/vocab.',
      )
      parser.add_argument(
          '--tf_checkpoint_path',
          default='',
          type=str,
          help='An optional path to a TensorFlow checkpoint path to be converted.',
      )
      parser.add_argument(
          '--transfo_xl_config_file',
          default='',
          type=str,
          help=(
              'An optional config json file corresponding to the pre-trained BERT model. \n'
              'This specifies the model architecture.'
          ),
      )
      parser.add_argument(
          '--transfo_xl_dataset_file',
          default='',
          type=str,
          help='An optional dataset file to be converted in a vocabulary.',
      )
      __lowerCamelCase						: Optional[int]		=   parser.parse_args()
      convert_transfo_xl_checkpoint_to_pytorch(
          args.tf_checkpoint_path,
          args.transfo_xl_config_file,
          args.pytorch_dump_folder_path,
          args.transfo_xl_dataset_file,
      )






