


import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


lowerCAmelCase       			=						'<<<<<<< This should probably be modified because it mentions: '

lowerCAmelCase       			=						'=======\n>>>>>>>\n'

lowerCAmelCase       			=						[
    'TextEncoderConfig',
    'ByteTextEncoder',
    'SubwordTextEncoder',
    'encoder_config',
    'maybe_build_from_corpus',
    'manual_dir',
]

lowerCAmelCase       			=						[
    # (pattern, replacement)
    # Order is important here for some replacements
    (r'tfds\.core', r'datasets'),
    (r'tf\.io\.gfile\.GFile', r'open'),
    (r'tf\.([\w\d]+)', r'datasets.Value(\'\1\')'),
    (r'tfds\.features\.Text\(\)', r'datasets.Value(\'string\')'),
    (r'tfds\.features\.Text\(', r'datasets.Value(\'string\'),'),
    (r'features\s*=\s*tfds.features.FeaturesDict\(', r'features=datasets.Features('),
    (r'tfds\.features\.FeaturesDict\(', r'dict('),
    (r'The TensorFlow Datasets Authors', r'The TensorFlow Datasets Authors and the HuggingFace Datasets Authors'),
    (r'tfds\.', r'datasets.'),
    (r'dl_manager\.manual_dir', r'self.config.data_dir'),
    (r'self\.builder_config', r'self.config'),
]







def 			_lowerCamelCase(      lowercase__      )		->      List[str]:

		'''simple docstring'''







		return ConvertCommand(args.tfds_path  ,					args.datasets_directory      )





class  A       (  A_					):


							@staticmethod
							def      _A			(lowerCAmelCase   ):
									__lowercase=     parser.add_parser(
									    'convert' ,							help='Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.' ,							)
									train_parser.add_argument(
									    '--tfds_path' ,							type=A__ ,							required=A__ ,							help='Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.' ,							)
									train_parser.add_argument(
									    '--datasets_directory' ,							type=A__ ,							required=A__ ,							help='Path to the HuggingFace Datasets folder.'   )
									train_parser.set_defaults(func=A__   )


							def __init__(self ,							lowerCAmelCase ,							lowerCAmelCase ,							*lowerCAmelCase   ):
									__lowercase=     get_logger('datasets-cli/converting'   )

									__lowercase=     tfds_path
									__lowercase=     datasets_directory







							def      _A			(self   ):
									if os.path.isdir(self._tfds_path   ):
											__lowercase=     os.path.abspath(self._tfds_path   )
									elif os.path.isfile(self._tfds_path   ):
											__lowercase=     os.path.dirname(self._tfds_path   )
									else:
											raise ValueError('--tfds_path is neither a directory nor a file. Please check path.'   )

									__lowercase=     os.path.abspath(self._datasets_directory   )

									self._logger.info(f'Converting datasets from {abs_tfds_path} to {abs_datasets_path}'   )

									__lowercase=     []
									__lowercase=     []
									__lowercase=     {}

									if os.path.isdir(self._tfds_path   ):
											__lowercase=     os.listdir(A__   )
									else:
											__lowercase=     [os.path.basename(self._tfds_path   )]

									for f_name in file_names:
											self._logger.info(f'Looking at file {f_name}'   )
											__lowercase=     os.path.join(A__ ,							A__   )
											__lowercase=     os.path.join(A__ ,							A__   )

											if not os.path.isfile(A__   ) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
													self._logger.info('Skipping file'   )
													continue

											with open(A__ ,							encoding='utf-8'   ) as f:
													__lowercase=     f.readlines()

											__lowercase=     []
											__lowercase=     False
											__lowercase=     False
											__lowercase=     []
											for line in lines:
													__lowercase=     line

													# Convert imports
													if "import tensorflow.compat.v2 as tf" in out_line:
															continue
													elif "@tfds.core" in out_line:
															continue
													elif "builder=self" in out_line:
															continue
													elif "import tensorflow_datasets.public_api as tfds" in out_line:
															__lowercase=     """import datasets\n"""
													elif "import tensorflow" in out_line:
															# order is important here
															__lowercase=     """"""
															continue
													elif "from absl import logging" in out_line:
															__lowercase=     """from datasets import logging\n"""
													elif "getLogger" in out_line:
															__lowercase=     out_line.replace('getLogger' ,							'get_logger'   )
													elif any(expression in out_line for expression in TO_HIGHLIGHT   ):
															__lowercase=     True
															__lowercase=     list(filter(lambda lowerCAmelCase   : e in out_line ,							A__   )   )
															out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(A__   ) + '\n'   )
															out_lines.append(A__   )
															out_lines.append(A__   )
															continue
													else:
															for pattern, replacement in TO_CONVERT:
																	__lowercase=     re.sub(A__ ,							A__ ,							A__   )

                # Take care of saving utilities (to later move them together with main script)
													if "tensorflow_datasets" in out_line:
															__lowercase=     re.match(r'from\stensorflow_datasets.*import\s([^\.\r\n]+)' ,							A__   )
															tfds_imports.extend(imp.strip() for imp in match.group(1   ).split(','   )   )
															__lowercase=     """from . import """ + match.group(1   )

													# Check we have not forget anything
													if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
															raise ValueError(f'Error converting {out_line.strip()}'   )

													if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
															__lowercase=     True
													out_lines.append(A__   )

											if is_builder or "wmt" in f_name:
													# We create a new directory for each dataset
													__lowercase=     f_name.replace('.py' ,							''   )
													__lowercase=     os.path.join(A__ ,							A__   )
													__lowercase=     os.path.join(A__ ,							A__   )
													os.makedirs(A__ ,							exist_ok=A__   )
													self._logger.info(f'Adding directory {output_dir}'   )
													imports_to_builder_map.update({imp: output_dir for imp in tfds_imports}   )
											else:
													# Utilities will be moved at the end
													utils_files.append(A__   )

											if needs_manual_update:
													with_manual_update.append(A__   )

											with open(A__ ,							'w' ,							encoding='utf-8'   ) as f:
													f.writelines(A__   )
											self._logger.info(f'Converted in {output_file}'   )

									for utils_file in utils_files:
											try:
													__lowercase=     os.path.basename(A__   )
													__lowercase=     imports_to_builder_map[f_name.replace('.py' ,							''   )]
													self._logger.info(f'Moving {dest_folder} to {utils_file}'   )
													shutil.copy(A__ ,							A__   )
											except KeyError:
													self._logger.error(f'Cannot find destination folder for {utils_file}. Please copy manually.'   )

									if with_manual_update:
											for file_path in with_manual_update:
													self._logger.warning(
													    f'You need to manually update file {file_path} to remove configurations using \'TextEncoderConfig\'.'   )









import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
	from sagemaker import Session, TrainingJobAnalytics
	from sagemaker.huggingface import HuggingFace



@pytest.mark.skipif(
    literal_eval(os.getenv('''TEST_SAGEMAKER'''						,  '''False'''							)							) is not True						,  reason='''Skipping test because should only be run when releasing minor transformers version'''						,  )
@pytest.mark.usefixtures('''sm_env'''							)
@parameterized_class(
    [
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_glue.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.p3.16xlarge''',
            '''results''': {'''train_runtime''': 6_50, '''eval_accuracy''': 0.7, '''eval_loss''': 0.6},
        },
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_ddp.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.p3.16xlarge''',
            '''results''': {'''train_runtime''': 6_00, '''eval_accuracy''': 0.7, '''eval_loss''': 0.6},
        },
        {
            '''framework''': '''tensorflow''',
            '''script''': '''run_tf_dist.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.p3.16xlarge''',
            '''results''': {'''train_runtime''': 6_00, '''eval_accuracy''': 0.6, '''eval_loss''': 0.7},
        },
    ]							)
class        _a		(unittest.TestCase							):


					'''simple docstring'''






					def     __A     (					self     ):
						if self.framework == "pytorch":
							subprocess.run(
							    F"""cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py""".split()       ,			encoding="""utf-8"""       ,			check=A__       ,			)
						assert hasattr(self       ,			"""env"""     )






					def     __A     (					self       ,			A__     ):
						A__				:       int	  =						F"""{self.env.base_job_name}-{instance_count}-{'ddp' if 'ddp' in self.script else 'smd'}"""
						# distributed data settings
						A__				:       str	  =						{"""smdistributed""": {"""dataparallel""": {"""enabled""": True}}} if self.script != """run_ddp.py""" else None

						# creates estimator
						return HuggingFace(
						    entry_point=self.script       ,			source_dir=self.env.test_path       ,			role=self.env.role       ,			image_uri=self.env.image_uri       ,			base_job_name=A__       ,			instance_count=A__       ,			instance_type=self.instance_type       ,			debugger_hook_config=A__       ,			hyperparameters={**self.env.distributed_hyperparameters, """model_name_or_path""": self.model_name_or_path}       ,			metric_definitions=self.env.metric_definitions       ,			distribution=A__       ,			py_version="""py36"""       ,			)






					def     __A     (					self       ,			A__     ):
						TrainingJobAnalytics(A__     ).export_csv(F"""{self.env.test_path}/{job_name}_metrics.csv"""     )






					@parameterized.expand([(2,)]     )
					def     __A     (					self       ,			A__     ):
						# create estimator
						A__				:       str	  =						self.create_estimator(A__     )

						# run training
						estimator.fit()

						# result dataframe
						A__				:       str	  =						TrainingJobAnalytics(estimator.latest_training_job.name     ).dataframe()

						# extract kpis
						A__				:       List[Any]	  =						list(result_metrics_df[result_metrics_df.metric_name == """eval_accuracy"""]["""value"""]     )
						A__				:       Optional[Any]	  =						list(result_metrics_df[result_metrics_df.metric_name == """eval_loss"""]["""value"""]     )
						# get train time from SageMaker job, this includes starting, preprocessing, stopping
						A__				:       str	  =						(
						    Session().describe_training_job(estimator.latest_training_job.name     ).get("""TrainingTimeInSeconds"""       ,			99_9999     )
						)

						# assert kpis
						assert train_runtime <= self.results["train_runtime"]
						assert all(t >= self.results["""eval_accuracy"""] for t in eval_accuracy     )
						assert all(t <= self.results["""eval_loss"""] for t in eval_loss     )

						# dump tests result into json file to share in PR
						with open(F"""{estimator.latest_training_job.name}.json"""       ,			"""w"""     ) as outfile:
							json.dump({"""train_time""": train_runtime, """eval_accuracy""": eval_accuracy, """eval_loss""": eval_loss}       ,			A__     )



