






import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


__magic_name__   :  int        =							'''<<<<<<< This should probably be modified because it mentions: '''

__magic_name__   :  Optional[int]        =							'''=======
>>>>>>>
'''

__magic_name__   :  List[str]        =							[
    '''TextEncoderConfig''',
    '''ByteTextEncoder''',
    '''SubwordTextEncoder''',
    '''encoder_config''',
    '''maybe_build_from_corpus''',
    '''manual_dir''',
]

__magic_name__   :  Dict        =							[
    # (pattern, replacement)
    # Order is important here for some replacements
    (r'''tfds\.core''', r'''datasets'''),
    (r'''tf\.io\.gfile\.GFile''', r'''open'''),
    (r'''tf\.([\w\d]+)''', r'''datasets.Value(\'\1\')'''),
    (r'''tfds\.features\.Text\(\)''', r'''datasets.Value(\'string\')'''),
    (r'''tfds\.features\.Text\(''', r'''datasets.Value(\'string\'),'''),
    (r'''features\s*=\s*tfds.features.FeaturesDict\(''', r'''features=datasets.Features('''),
    (r'''tfds\.features\.FeaturesDict\(''', r'''dict('''),
    (r'''The TensorFlow Datasets Authors''', r'''The TensorFlow Datasets Authors and the HuggingFace Datasets Authors'''),
    (r'''tfds\.''', r'''datasets.'''),
    (r'''dl_manager\.manual_dir''', r'''self.config.data_dir'''),
    (r'''self\.builder_config''', r'''self.config'''),
]
def 		lowercase__  (				_UpperCamelCase) ->			List[Any]:
      """simple docstring"""

      return ConvertCommand(args.tfds_path       ,							args.datasets_directory)





class 					A__     (     __snake_case ):

     '''simple docstring'''






     @staticmethod
     def   _SCREAMING_SNAKE_CASE     ( _SCREAMING_SNAKE_CASE      :    ArgumentParser ):






           """simple docstring"""




           UpperCamelCase					       =  parser.add_parser(
               'convert'   ,						help='Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.'   ,						)
           train_parser.add_argument(
               '--tfds_path'   ,						type=_SCREAMING_SNAKE_CASE   ,						required=_SCREAMING_SNAKE_CASE   ,						help='Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.'   ,						)
           train_parser.add_argument(
               '--datasets_directory'   ,						type=_SCREAMING_SNAKE_CASE   ,						required=_SCREAMING_SNAKE_CASE   ,						help='Path to the HuggingFace Datasets folder.' )
           train_parser.set_defaults(func=_SCREAMING_SNAKE_CASE )







     def __init__( self      :    List[str]   ,						_SCREAMING_SNAKE_CASE      :    str   ,						_SCREAMING_SNAKE_CASE      :    str   ,						*_SCREAMING_SNAKE_CASE      :    List[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  get_logger('datasets-cli/converting' )

           UpperCamelCase					       =  tfds_path
           UpperCamelCase					       =  datasets_directory







     def   _SCREAMING_SNAKE_CASE     ( self      :    Dict ):






           """simple docstring"""




           if os.path.isdir(self._tfds_path ):
                 UpperCamelCase					       =  os.path.abspath(self._tfds_path )
           elif os.path.isfile(self._tfds_path ):
                 UpperCamelCase					       =  os.path.dirname(self._tfds_path )
           else:
                 raise ValueError('--tfds_path is neither a directory nor a file. Please check path.' )

           UpperCamelCase					       =  os.path.abspath(self._datasets_directory )

           self._logger.info(f'Converting datasets from {abs_tfds_path} to {abs_datasets_path}' )

           UpperCamelCase					       =  []
           UpperCamelCase					       =  []
           UpperCamelCase					       =  {}

           if os.path.isdir(self._tfds_path ):
                 UpperCamelCase					       =  os.listdir(_SCREAMING_SNAKE_CASE )
           else:
                 UpperCamelCase					       =  [os.path.basename(self._tfds_path )]

           for f_name in file_names:
                 self._logger.info(f'Looking at file {f_name}' )
                 UpperCamelCase					       =  os.path.join(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
                 UpperCamelCase					       =  os.path.join(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

                 if not os.path.isfile(_SCREAMING_SNAKE_CASE ) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
                       self._logger.info('Skipping file' )
                       continue

                 with open(_SCREAMING_SNAKE_CASE   ,						encoding='utf-8' ) as f:
                       UpperCamelCase					       =  f.readlines()

                 UpperCamelCase					       =  []
                 UpperCamelCase					       =  False
                 UpperCamelCase					       =  False
                 UpperCamelCase					       =  []
                 for line in lines:
                       UpperCamelCase					       =  line

                       # Convert imports
                       if "import tensorflow.compat.v2 as tf" in out_line:
                             continue
                       elif "@tfds.core" in out_line:
                             continue
                       elif "builder=self" in out_line:
                             continue
                       elif "import tensorflow_datasets.public_api as tfds" in out_line:
                             UpperCamelCase					       =  'import datasets\n'
                       elif "import tensorflow" in out_line:
                             # order is important here
                             UpperCamelCase					       =  ''
                             continue
                       elif "from absl import logging" in out_line:
                             UpperCamelCase					       =  'from datasets import logging\n'
                       elif "getLogger" in out_line:
                             UpperCamelCase					       =  out_line.replace('getLogger'   ,						'get_logger' )
                       elif any(expression in out_line for expression in TO_HIGHLIGHT ):
                             UpperCamelCase					       =  True
                             UpperCamelCase					       =  list(filter(lambda _SCREAMING_SNAKE_CASE : e in out_line   ,						_SCREAMING_SNAKE_CASE ) )
                             out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(_SCREAMING_SNAKE_CASE ) + '\n' )
                             out_lines.append(_SCREAMING_SNAKE_CASE )
                             out_lines.append(_SCREAMING_SNAKE_CASE )
                             continue
                       else:
                             for pattern, replacement in TO_CONVERT:
                                   UpperCamelCase					       =  re.sub(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

                # Take care of saving utilities (to later move them together with main script)
                       if "tensorflow_datasets" in out_line:
                             UpperCamelCase					       =  re.match(r'from\stensorflow_datasets.*import\s([^\.\r\n]+)'   ,						_SCREAMING_SNAKE_CASE )
                             tfds_imports.extend(imp.strip() for imp in match.group(1 ).split(',' ) )
                             UpperCamelCase					       =  'from . import ' + match.group(1 )

                       # Check we have not forget anything
                       if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
                             raise ValueError(f'Error converting {out_line.strip()}' )

                       if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
                             UpperCamelCase					       =  True
                       out_lines.append(_SCREAMING_SNAKE_CASE )

                 if is_builder or "wmt" in f_name:
                       # We create a new directory for each dataset
                       UpperCamelCase					       =  f_name.replace('.py'   ,						'' )
                       UpperCamelCase					       =  os.path.join(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  os.path.join(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
                       os.makedirs(_SCREAMING_SNAKE_CASE   ,						exist_ok=_SCREAMING_SNAKE_CASE )
                       self._logger.info(f'Adding directory {output_dir}' )
                       imports_to_builder_map.update({imp: output_dir for imp in tfds_imports} )
                 else:
                       # Utilities will be moved at the end
                       utils_files.append(_SCREAMING_SNAKE_CASE )

                 if needs_manual_update:
                       with_manual_update.append(_SCREAMING_SNAKE_CASE )

                 with open(_SCREAMING_SNAKE_CASE   ,						'w'   ,						encoding='utf-8' ) as f:
                       f.writelines(_SCREAMING_SNAKE_CASE )
                 self._logger.info(f'Converted in {output_file}' )

           for utils_file in utils_files:
                 try:
                       UpperCamelCase					       =  os.path.basename(_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  imports_to_builder_map[f_name.replace('.py'   ,						'' )]
                       self._logger.info(f'Moving {dest_folder} to {utils_file}' )
                       shutil.copy(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
                 except KeyError:
                       self._logger.error(f'Cannot find destination folder for {utils_file}. Please copy manually.' )

           if with_manual_update:
                 for file_path in with_manual_update:
                       self._logger.warning(
                           f'You need to manually update file {file_path} to remove configurations using \'TextEncoderConfig\'.' )











import re

import jax.numpy as jnp
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.random import PRNGKey

from ..utils import logging


__magic_name__   :  Optional[int]        =							logging.get_logger(__name__)
def 		lowercase__  (				_UpperCamelCase) ->			Dict:
      """simple docstring"""

      UpperCamelCase					       =  r'\w+[.]\d+'
      UpperCamelCase					       =  re.findall(_UpperCamelCase       ,							_UpperCamelCase)
      for pat in pats:
            UpperCamelCase					       =  key.replace(_UpperCamelCase       ,							'_'.join(pat.split('.')))
      return key
def 		lowercase__  (				_UpperCamelCase       ,							_UpperCamelCase       ,							_UpperCamelCase) ->			Tuple:
      """simple docstring"""

      UpperCamelCase					       =  pt_tuple_key[:-1] + ('scale',)
      if (
          any('norm' in str_ for str_ in pt_tuple_key)
          and (pt_tuple_key[-1] == "bias")
          and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict)
          and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict)
      ):
            UpperCamelCase					       =  pt_tuple_key[:-1] + ('scale',)
            return renamed_pt_tuple_key, pt_tensor
      elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict:
            UpperCamelCase					       =  pt_tuple_key[:-1] + ('scale',)
            return renamed_pt_tuple_key, pt_tensor

      # embedding
      if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict:
            UpperCamelCase					       =  pt_tuple_key[:-1] + ('embedding',)
            return renamed_pt_tuple_key, pt_tensor

      # conv layer
      UpperCamelCase					       =  pt_tuple_key[:-1] + ('kernel',)
      if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4:
            UpperCamelCase					       =  pt_tensor.transpose(2       ,							3       ,							1       ,							0)
            return renamed_pt_tuple_key, pt_tensor

      # linear layer
      UpperCamelCase					       =  pt_tuple_key[:-1] + ('kernel',)
      if pt_tuple_key[-1] == "weight":
            UpperCamelCase					       =  pt_tensor.T
            return renamed_pt_tuple_key, pt_tensor

      # old PyTorch layer norm weight
      UpperCamelCase					       =  pt_tuple_key[:-1] + ('weight',)
      if pt_tuple_key[-1] == "gamma":
            return renamed_pt_tuple_key, pt_tensor

      # old PyTorch layer norm bias
      UpperCamelCase					       =  pt_tuple_key[:-1] + ('bias',)
      if pt_tuple_key[-1] == "beta":
            return renamed_pt_tuple_key, pt_tensor

      return pt_tuple_key, pt_tensor
def 		lowercase__  (				_UpperCamelCase       ,							_UpperCamelCase       ,							_UpperCamelCase=42) ->			Optional[Any]:
      """simple docstring"""

      UpperCamelCase					       =  {k: v.numpy() for k, v in pt_state_dict.items()}

      # Step 2: Since the model is stateless, get random Flax params
      UpperCamelCase					       =  flax_model.init_weights(PRNGKey(_UpperCamelCase))

      UpperCamelCase					       =  flatten_dict(_UpperCamelCase)
      UpperCamelCase					       =  {}

      # Need to change some parameters name to match Flax names
      for pt_key, pt_tensor in pt_state_dict.items():
            UpperCamelCase					       =  rename_key(_UpperCamelCase)
            UpperCamelCase					       =  tuple(renamed_pt_key.split('.'))

            # Correctly rename weight parameters
            UpperCamelCase       ,  UpperCamelCase					       =  rename_key_and_reshape_tensor(_UpperCamelCase       ,							_UpperCamelCase       ,							_UpperCamelCase)

            if flax_key in random_flax_state_dict:
                  if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
                        raise ValueError(
                            F'PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '
                            F'{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.')

        # also add unexpected weight so that warning is thrown
            UpperCamelCase					       =  jnp.asarray(_UpperCamelCase)

      return unflatten_dict(_UpperCamelCase)



