
'''simple docstring'''



import inspect
import os
from argparse import ArgumentParser, Namespace
from importlib import import_module

import huggingface_hub
import numpy as np
from packaging import version

from .. import (
    FEATURE_EXTRACTOR_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoImageProcessor,
    AutoProcessor,
    AutoTokenizer,
    is_datasets_available,
    is_tf_available,
    is_torch_available,
)
from ..utils import TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, logging
from . import BaseTransformersCLICommand


if is_tf_available():
     import tensorflow as tf

     tf.config.experimental.enable_tensor_float_aa_execution(False)

if is_torch_available():
     import torch

if is_datasets_available():
     from datasets import load_dataset


UpperCamelCase__					:						List[str]											=     5E-5  # larger error tolerance than in our internal tests, to avoid flaky user-facing errors
def       lowerCAmelCase_		( SCREAMING_SNAKE_CASE_  )   ->   Union[str, Any]:




   """simple docstring"""
   return PTtoTFCommand(
       args.model_name			,					args.local_dir			,					args.max_error			,					args.new_weights			,					args.no_pr			,					args.push			,					args.extra_commit_description			,					args.override_model_class			,					)







class       _a (_lowerCamelCase):





       """simple docstring"""




       @staticmethod
       def 				UpperCamelCase (							A__					)			->	List[str]:
          _SCREAMING_SNAKE_CASE          = parser.add_parser(
              """pt-to-tf"""				,							help=(
                  """CLI tool to run convert a transformers model from a PyTorch checkpoint to a TensorFlow checkpoint."""
                  """ Can also be used to validate existing weights without opening PRs, with --no-pr."""
              )				,							)
          train_parser.add_argument(
              """--model-name"""				,							type=A__				,							required=A__				,							help="""The model name, including owner/organization, as seen on the hub."""				,							)
          train_parser.add_argument(
              """--local-dir"""				,							type=A__				,							default=""""""				,							help="""Optional local directory of the model repository. Defaults to /tmp/{model_name}"""				,							)
          train_parser.add_argument(
              """--max-error"""				,							type=A__				,							default=A__				,							help=(
                  F"Maximum error tolerance. Defaults to {MAX_ERROR}. This flag should be avoided, use at your own risk."
              )				,							)
          train_parser.add_argument(
              """--new-weights"""				,							action="""store_true"""				,							help="""Optional flag to create new TensorFlow weights, even if they already exist."""				,							)
          train_parser.add_argument(
              """--no-pr"""				,							action="""store_true"""				,							help="""Optional flag to NOT open a PR with converted weights."""					)
          train_parser.add_argument(
              """--push"""				,							action="""store_true"""				,							help="""Optional flag to push the weights directly to `main` (requires permissions)"""				,							)
          train_parser.add_argument(
              """--extra-commit-description"""				,							type=A__				,							default=""""""				,							help="""Optional additional commit description to use when opening a PR (e.g. to tag the owner)."""				,							)
          train_parser.add_argument(
              """--override-model-class"""				,							type=A__				,							default=A__				,							help="""If you think you know better than the auto-detector, you can specify the model class here. """
              """Can be either an AutoModel class or a specific model class like BertForSequenceClassification."""				,							)
          train_parser.set_defaults(func=A__					)






       @staticmethod
       def 				UpperCamelCase (							A__				,							A__					)			->	str:
          _SCREAMING_SNAKE_CASE          = set(pt_outputs.keys()					)
          _SCREAMING_SNAKE_CASE          = set(tf_outputs.keys()					)
          if pt_out_attrs != tf_out_attrs:
             raise ValueError(
                 F"The model outputs have different attributes, aborting. (Pytorch: {pt_out_attrs}, TensorFlow:"
                 F" {tf_out_attrs})"					)

          # 2. For each output attribute, computes the difference
          def _find_pt_tf_differences(A__				,							A__				,							A__				,							A__=""					):
             # If the current attribute is a tensor, it is a leaf and we make the comparison. Otherwise, we will dig in
             # recursivelly, keeping the name of the attribute.
             if isinstance(A__				,							torch.Tensor					):
                _SCREAMING_SNAKE_CASE          = np.max(np.abs(pt_out.numpy() - tf_out.numpy()					)					)
                _SCREAMING_SNAKE_CASE          = tensor_difference
             else:
                _SCREAMING_SNAKE_CASE          = attr_name
                for i, pt_item in enumerate(A__					):
                   # If it is a named attribute, we keep the name. Otherwise, just its index.
                   if isinstance(A__				,							A__					):
                      _SCREAMING_SNAKE_CASE          = root_name + pt_item
                      _SCREAMING_SNAKE_CASE          = tf_out[pt_item]
                      _SCREAMING_SNAKE_CASE          = pt_out[pt_item]
                   else:
                      _SCREAMING_SNAKE_CASE          = root_name + F"[{i}]"
                      _SCREAMING_SNAKE_CASE          = tf_out[i]
                   _SCREAMING_SNAKE_CASE          = _find_pt_tf_differences(A__				,							A__				,							A__				,							A__					)

             return differences

          return _find_pt_tf_differences(A__				,							A__				,							{}					)






       def __init__(							self				,							A__				,							A__				,							A__				,							A__				,							A__				,							A__				,							A__				,							A__				,							*A__				,							)			->	List[str]:
          _SCREAMING_SNAKE_CASE          = logging.get_logger("""transformers-cli/pt_to_tf"""					)
          _SCREAMING_SNAKE_CASE          = model_name
          _SCREAMING_SNAKE_CASE          = local_dir if local_dir else os.path.join("""/tmp"""				,							A__					)
          _SCREAMING_SNAKE_CASE          = max_error
          _SCREAMING_SNAKE_CASE          = new_weights
          _SCREAMING_SNAKE_CASE          = no_pr
          _SCREAMING_SNAKE_CASE          = push
          _SCREAMING_SNAKE_CASE          = extra_commit_description
          _SCREAMING_SNAKE_CASE          = override_model_class






       def 				UpperCamelCase (							self				,							A__				,							A__				,							A__					)			->	Union[str, Any]:

          def _get_audio_input():
             _SCREAMING_SNAKE_CASE          = load_dataset("""hf-internal-testing/librispeech_asr_dummy"""				,							"""clean"""				,							split="""validation"""					)
             _SCREAMING_SNAKE_CASE          = ds.sort("""id"""					).select(range(2					)					)[:2]["""audio"""]
             _SCREAMING_SNAKE_CASE          = [x["""array"""] for x in speech_samples]
             return raw_samples

          _SCREAMING_SNAKE_CASE          = type(pt_model.config					)
          if model_config_class in PROCESSOR_MAPPING:
             _SCREAMING_SNAKE_CASE          = AutoProcessor.from_pretrained(self._local_dir					)
             if model_config_class in TOKENIZER_MAPPING and processor.tokenizer.pad_token is None:
                _SCREAMING_SNAKE_CASE          = processor.tokenizer.eos_token
          elif model_config_class in IMAGE_PROCESSOR_MAPPING:
             _SCREAMING_SNAKE_CASE          = AutoImageProcessor.from_pretrained(self._local_dir					)
          elif model_config_class in FEATURE_EXTRACTOR_MAPPING:
             _SCREAMING_SNAKE_CASE          = AutoFeatureExtractor.from_pretrained(self._local_dir					)
          elif model_config_class in TOKENIZER_MAPPING:
             _SCREAMING_SNAKE_CASE          = AutoTokenizer.from_pretrained(self._local_dir					)
             if processor.pad_token is None:
                _SCREAMING_SNAKE_CASE          = processor.eos_token
          else:
             raise ValueError(F"Unknown data processing type (model config type: {model_config_class})"					)

          _SCREAMING_SNAKE_CASE          = set(inspect.signature(pt_model.forward					).parameters.keys()					)
          _SCREAMING_SNAKE_CASE          = {}
          if "input_ids" in model_forward_signature:
             processor_inputs.update(
                 {
                     """text""": ["""Hi there!""", """I am a batch with more than one row and different input lengths."""],
                     """padding""": True,
                     """truncation""": True,
                 }					)
          if "pixel_values" in model_forward_signature:
             _SCREAMING_SNAKE_CASE          = load_dataset("""cifar10"""				,							"""plain_text"""				,							split="""test"""					)[:2]["""img"""]
             processor_inputs.update({"""images""": sample_images}					)
          if "input_features" in model_forward_signature:
             _SCREAMING_SNAKE_CASE          = inspect.signature(processor.feature_extractor					).parameters
             # Pad to the largest input length by default but take feature extractor default
             # padding value if it exists e.g. "max_length" and is not False or None
             if "padding" in feature_extractor_signature:
                _SCREAMING_SNAKE_CASE          = feature_extractor_signature["""padding"""].default
                if default_strategy is not False and default_strategy is not None:
                   _SCREAMING_SNAKE_CASE          = default_strategy
                else:
                   _SCREAMING_SNAKE_CASE          = True
             else:
                _SCREAMING_SNAKE_CASE          = True
             processor_inputs.update({"""audio""": _get_audio_input(), """padding""": padding_strategy}					)
          if "input_values" in model_forward_signature:  # Wav2Vec2 audio input
             processor_inputs.update({"""audio""": _get_audio_input(), """padding""": True}					)
          _SCREAMING_SNAKE_CASE          = processor(**A__				,							return_tensors="""pt"""					)
          _SCREAMING_SNAKE_CASE          = processor(**A__				,							return_tensors="""tf"""					)

          # Extra input requirements, in addition to the input modality
          if (
              config.is_encoder_decoder
              or (hasattr(A__				,							"""encoder"""					) and hasattr(A__				,							"""decoder"""					))
              or "decoder_input_ids" in tf_dummy_inputs
          ):
             _SCREAMING_SNAKE_CASE          = np.asarray([[1], [1]]				,							dtype=A__					) * (pt_model.config.decoder_start_token_id or 0)
             pt_input.update({"""decoder_input_ids""": torch.tensor(A__					)}					)
             tf_input.update({"""decoder_input_ids""": tf.convert_to_tensor(A__					)}					)

          return pt_input, tf_input






       def 				UpperCamelCase (							self					)			->	Tuple:
          # hub version 0.9.0 introduced the possibility of programmatically opening PRs with normal write tokens.
          if version.parse(huggingface_hub.__version__					) < version.parse("""0.9.0"""					):
             raise ImportError(
                 """The huggingface_hub version must be >= 0.9.0 to use this command. Please update your huggingface_hub"""
                 """ installation."""					)
          else:
             from huggingface_hub import Repository, create_commit
             from huggingface_hub._commit_api import CommitOperationAdd

          # Fetch remote data
          _SCREAMING_SNAKE_CASE          = Repository(local_dir=self._local_dir				,							clone_from=self._model_name					)

          # Load config and get the appropriate architecture -- the latter is needed to convert the head's weights
          _SCREAMING_SNAKE_CASE          = AutoConfig.from_pretrained(self._local_dir					)
          _SCREAMING_SNAKE_CASE          = config.architectures
          if self._override_model_class is not None:
             if self._override_model_class.startswith("""TF"""					):
                _SCREAMING_SNAKE_CASE          = [self._override_model_class[2:]]
             else:
                _SCREAMING_SNAKE_CASE          = [self._override_model_class]
             try:
                _SCREAMING_SNAKE_CASE          = getattr(import_module("""transformers"""					)				,							architectures[0]					)
             except AttributeError:
                raise ValueError(F"Model class {self._override_model_class} not found in transformers."					)
             try:
                _SCREAMING_SNAKE_CASE          = getattr(import_module("""transformers"""					)				,							"""TF""" + architectures[0]					)
             except AttributeError:
                raise ValueError(F"TF model class TF{self._override_model_class} not found in transformers."					)
          elif architectures is None:  # No architecture defined -- use auto classes
             _SCREAMING_SNAKE_CASE          = getattr(import_module("""transformers"""					)				,							"""AutoModel"""					)
             _SCREAMING_SNAKE_CASE          = getattr(import_module("""transformers"""					)				,							"""TFAutoModel"""					)
             self._logger.warning("""No detected architecture, using AutoModel/TFAutoModel"""					)
          else:  # Architecture defined -- use it
             if len(A__					) > 1:
                raise ValueError(F"More than one architecture was found, aborting. (architectures = {architectures})"					)
             self._logger.warning(F"Detected architecture: {architectures[0]}"					)
             _SCREAMING_SNAKE_CASE          = getattr(import_module("""transformers"""					)				,							architectures[0]					)
             try:
                _SCREAMING_SNAKE_CASE          = getattr(import_module("""transformers"""					)				,							"""TF""" + architectures[0]					)
             except AttributeError:
                raise AttributeError(F"The TensorFlow equivalent of {architectures[0]} doesn't exist in transformers."					)

        # Check the TF dummy inputs to see what keys we need in the forward pass
          _SCREAMING_SNAKE_CASE          = tf_class.from_config(A__					)
          _SCREAMING_SNAKE_CASE          = tf_from_pt_model.dummy_inputs

          del tf_from_pt_model  # Try to keep only one model in memory at a time

          # Load the model and get some basic inputs
          _SCREAMING_SNAKE_CASE          = pt_class.from_pretrained(self._local_dir					)
          pt_model.eval()

          _SCREAMING_SNAKE_CASE       ,							_SCREAMING_SNAKE_CASE          = self.get_inputs(A__				,							A__				,							A__					)

          with torch.no_grad():
             _SCREAMING_SNAKE_CASE          = pt_model(**A__				,							output_hidden_states=A__					)
          del pt_model  # will no longer be used, and may have a large memory footprint

          _SCREAMING_SNAKE_CASE          = tf_class.from_pretrained(self._local_dir				,							from_pt=A__					)
          _SCREAMING_SNAKE_CASE          = tf_from_pt_model(**A__				,							output_hidden_states=A__				,							training=A__					)

          # Confirms that cross loading PT weights into TF worked.
          _SCREAMING_SNAKE_CASE          = self.find_pt_tf_differences(A__				,							A__					)
          _SCREAMING_SNAKE_CASE          = {k: v for k, v in crossload_differences.items() if """hidden""" not in k}
          _SCREAMING_SNAKE_CASE          = {k: v for k, v in crossload_differences.items() if """hidden""" in k}
          if len(A__					) == 0 and architectures is not None:
             raise ValueError(
                 F"Something went wrong -- the config file has architectures ({architectures}), but no model head"
                 """ output was found. All outputs start with 'hidden'"""					)
          _SCREAMING_SNAKE_CASE          = max(output_differences.values()					) if output_differences else 0.0
          _SCREAMING_SNAKE_CASE          = max(hidden_differences.values()					)
          if max_crossload_output_diff > self._max_error or max_crossload_hidden_diff > self._max_error:
             raise ValueError(
                 """The cross-loaded TensorFlow model has different outputs, something went wrong!\n"""
                 + F"\nList of maximum output differences above the threshold ({self._max_error}):\n"
                 + """\n""".join([F"{k}: {v:.3e}" for k, v in output_differences.items() if v > self._max_error]					)
                 + F"\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n"
                 + """\n""".join([F"{k}: {v:.3e}" for k, v in hidden_differences.items() if v > self._max_error]					)					)

          # Save the weights in a TF format (if needed) and confirms that the results are still good
          _SCREAMING_SNAKE_CASE          = os.path.join(self._local_dir				,							A__					)
          _SCREAMING_SNAKE_CASE          = os.path.join(self._local_dir				,							A__					)
          if (not os.path.exists(A__					) and not os.path.exists(A__					)) or self._new_weights:
             tf_from_pt_model.save_pretrained(self._local_dir					)
          del tf_from_pt_model  # will no longer be used, and may have a large memory footprint

          _SCREAMING_SNAKE_CASE          = tf_class.from_pretrained(self._local_dir					)
          _SCREAMING_SNAKE_CASE          = tf_model(**A__				,							output_hidden_states=A__					)

          _SCREAMING_SNAKE_CASE          = self.find_pt_tf_differences(A__				,							A__					)
          _SCREAMING_SNAKE_CASE          = {k: v for k, v in conversion_differences.items() if """hidden""" not in k}
          _SCREAMING_SNAKE_CASE          = {k: v for k, v in conversion_differences.items() if """hidden""" in k}
          if len(A__					) == 0 and architectures is not None:
             raise ValueError(
                 F"Something went wrong -- the config file has architectures ({architectures}), but no model head"
                 """ output was found. All outputs start with 'hidden'"""					)
          _SCREAMING_SNAKE_CASE          = max(output_differences.values()					) if output_differences else 0.0
          _SCREAMING_SNAKE_CASE          = max(hidden_differences.values()					)
          if max_conversion_output_diff > self._max_error or max_conversion_hidden_diff > self._max_error:
             raise ValueError(
                 """The converted TensorFlow model has different outputs, something went wrong!\n"""
                 + F"\nList of maximum output differences above the threshold ({self._max_error}):\n"
                 + """\n""".join([F"{k}: {v:.3e}" for k, v in output_differences.items() if v > self._max_error]					)
                 + F"\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n"
                 + """\n""".join([F"{k}: {v:.3e}" for k, v in hidden_differences.items() if v > self._max_error]					)					)

          _SCREAMING_SNAKE_CASE          = """Update TF weights""" if self._new_weights else """Add TF weights"""
          if self._push:
             repo.git_add(auto_lfs_track=A__					)
             repo.git_commit(A__					)
             repo.git_push(blocking=A__					)  # this prints a progress bar with the upload
             self._logger.warning(F"TF weights pushed into {self._model_name}"					)
          elif not self._no_pr:
             self._logger.warning("""Uploading the weights into a new PR..."""					)
             _SCREAMING_SNAKE_CASE          = (
                 """Model converted by the [`transformers`' `pt_to_tf`"""
                 """ CLI](https://github.com/huggingface/transformers/blob/main/src/transformers/commands/pt_to_tf.py). """
                 """All converted model outputs and hidden layers were validated against its PyTorch counterpart.\n\n"""
                 F"Maximum crossload output difference={max_crossload_output_diff:.3e}; "
                 F"Maximum crossload hidden layer difference={max_crossload_hidden_diff:.3e};\n"
                 F"Maximum conversion output difference={max_conversion_output_diff:.3e}; "
                 F"Maximum conversion hidden layer difference={max_conversion_hidden_diff:.3e};\n"
             )
             if self._max_error > MAX_ERROR:
                commit_descrition += (
                    F"\n\nCAUTION: The maximum admissible error was manually increased to {self._max_error}!"
                )
             if self._extra_commit_description:
                commit_descrition += "\n\n" + self._extra_commit_description

             # sharded model -> adds all related files (index and .h5 shards)
             if os.path.exists(A__					):
                _SCREAMING_SNAKE_CASE          = [
                    CommitOperationAdd(path_in_repo=A__				,							path_or_fileobj=A__					)
                ]
                for shard_path in tf.io.gfile.glob(self._local_dir + """/tf_model-*.h5"""					):
                   operations += [
                       CommitOperationAdd(path_in_repo=os.path.basename(A__					)				,							path_or_fileobj=A__					)
                   ]
             else:
                _SCREAMING_SNAKE_CASE          = [CommitOperationAdd(path_in_repo=A__				,							path_or_fileobj=A__					)]

             _SCREAMING_SNAKE_CASE          = create_commit(
                 repo_id=self._model_name				,							operations=A__				,							commit_message=A__				,							commit_description=A__				,							repo_type="""model"""				,							create_pr=A__				,							).pr_url
             self._logger.warning(F"PR open in {hub_pr_url}"					)








'''simple docstring'''



import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
     import sqlitea

     import sqlalchemy


UpperCamelCase__					:						Tuple											=     datasets.utils.logging.get_logger(__name__)

@dataclass
class       _a (datasets.BuilderConfig):





       """simple docstring"""




       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     True
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     1_00_00
       SCREAMING_SNAKE_CASE					=     None






       def 				UpperCamelCase (							self					)			->	str:
          if self.sql is None:
             raise ValueError("""sql must be specified"""					)
          if self.con is None:
             raise ValueError("""con must be specified"""					)






       def 				UpperCamelCase (							self				,							A__				,							A__ = None				,							)			->	str:
          _SCREAMING_SNAKE_CASE          = config_kwargs.copy()
          # We need to stringify the Selectable object to make its hash deterministic

          # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
          _SCREAMING_SNAKE_CASE          = config_kwargs["""sql"""]
          if not isinstance(A__				,							A__					):
             if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
                import sqlalchemy

                if isinstance(A__				,							sqlalchemy.sql.Selectable					):
                   _SCREAMING_SNAKE_CASE          = sqlalchemy.create_engine(config_kwargs["""con"""].split("""://"""					)[0] + """://"""					)
                   _SCREAMING_SNAKE_CASE          = str(sql.compile(dialect=engine.dialect					)					)
                   _SCREAMING_SNAKE_CASE          = sql_str
                else:
                   raise TypeError(
                       F"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(A__					)}: {sql}"					)
             else:
                raise TypeError(
                    F"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(A__					)}: {sql}"					)
          _SCREAMING_SNAKE_CASE          = config_kwargs["""con"""]
          if not isinstance(A__				,							A__					):
             _SCREAMING_SNAKE_CASE          = id(A__					)
             logger.info(
                 F"SQL connection 'con' of type {type(A__					)} couldn't be hashed properly. To enable hashing, specify 'con' as URI string instead."					)

          return super().create_config_id(A__				,							custom_features=A__					)






       @property
       def 				UpperCamelCase (							self					)			->	Any:
          _SCREAMING_SNAKE_CASE          = {
              """index_col""": self.index_col,
              """columns""": self.columns,
              """params""": self.params,
              """coerce_float""": self.coerce_float,
              """parse_dates""": self.parse_dates,
          }
          return pd_read_sql_kwargs






class       _a (datasets.ArrowBasedBuilder):





       """simple docstring"""




       SCREAMING_SNAKE_CASE					=     SqlConfig






       def 				UpperCamelCase (							self					)			->	Optional[Any]:
          return datasets.DatasetInfo(features=self.config.features					)






       def 				UpperCamelCase (							self				,							A__					)			->	Dict:
          return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,							gen_kwargs={}					)]






       def 				UpperCamelCase (							self				,							A__					)			->	pa.Table:
          if self.config.features is not None:
             _SCREAMING_SNAKE_CASE          = self.config.features.arrow_schema
             if all(not require_storage_cast(A__					) for feature in self.config.features.values()					):
                # cheaper cast
                _SCREAMING_SNAKE_CASE          = pa.Table.from_arrays([pa_table[field.name] for field in schema]				,							schema=A__					)
             else:
                # more expensive cast; allows str <-> int/float or str to Audio for example
                _SCREAMING_SNAKE_CASE          = table_cast(A__				,							A__					)
          return pa_table






       def 				UpperCamelCase (							self					)			->	Optional[int]:
          _SCREAMING_SNAKE_CASE          = self.config.chunksize
          _SCREAMING_SNAKE_CASE          = pd.read_sql(
              self.config.sql				,							self.config.con				,							chunksize=A__				,							**self.config.pd_read_sql_kwargs					)
          _SCREAMING_SNAKE_CASE          = [sql_reader] if chunksize is None else sql_reader
          for chunk_idx, df in enumerate(A__					):
             _SCREAMING_SNAKE_CASE          = pa.Table.from_pandas(A__					)
             yield chunk_idx, self._cast_table(A__					)






