
"""simple docstring"""






import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


UpperCAmelCase        =				'<<<<<<< This should probably be modified because it mentions: '

UpperCAmelCase        =				'=======\n>>>>>>>\n'

UpperCAmelCase        =				[
    'TextEncoderConfig',
    'ByteTextEncoder',
    'SubwordTextEncoder',
    'encoder_config',
    'maybe_build_from_corpus',
    'manual_dir',
]

UpperCAmelCase        =				[
    # (pattern, replacement)
    # Order is important here for some replacements
    (r'tfds\.core', r'datasets'),
    (r'tf\.io\.gfile\.GFile', r'open'),
    (r'tf\.([\w\d]+)', r'datasets.Value(\'\1\')'),
    (r'tfds\.features\.Text\(\)', r'datasets.Value(\'string\')'),
    (r'tfds\.features\.Text\(', r'datasets.Value(\'string\'),'),
    (r'features\s*=\s*tfds.features.FeaturesDict\(', r'features=datasets.Features('),
    (r'tfds\.features\.FeaturesDict\(', r'dict('),
    (r'The TensorFlow Datasets Authors', r'The TensorFlow Datasets Authors and the HuggingFace Datasets Authors'),
    (r'tfds\.', r'datasets.'),
    (r'dl_manager\.manual_dir', r'self.config.data_dir'),
    (r'self\.builder_config', r'self.config'),
]




def __magic_name__	(				_lowerCamelCase:		Optional[Any]			)					->						Tuple:

	'''simple docstring'''




	return ConvertCommand(args.tfds_path,					args.datasets_directory			)



class 			lowercase					( _snake_case		):






			@staticmethod
			def 					UpperCAmelCase							(SCREAMING_SNAKE_CASE_  :	List[Any]			)       ->	Optional[int]:





				"""simple docstring"""


				lowerCAmelCase												=  parser.add_parser(
				    '''convert'''      ,help='''Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.'''      ,)
				train_parser.add_argument(
				    '''--tfds_path'''      ,type=__snake_case      ,required=__snake_case      ,help='''Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.'''      ,)
				train_parser.add_argument(
				    '''--datasets_directory'''      ,type=__snake_case      ,required=__snake_case      ,help='''Path to the HuggingFace Datasets folder.'''			)
				train_parser.set_defaults(func=__snake_case			)






			def __init__(self  :	Any      ,SCREAMING_SNAKE_CASE_  :	Optional[Any]      ,SCREAMING_SNAKE_CASE_  :	int      ,*SCREAMING_SNAKE_CASE_  :	Optional[int]			)       ->	Any:





				"""simple docstring"""


				lowerCAmelCase												=  get_logger('''datasets-cli/converting'''			)

				lowerCAmelCase												=  tfds_path
				lowerCAmelCase												=  datasets_directory






			def 					UpperCAmelCase							(self  :	Union[str, Any]			)       ->	Dict:





				"""simple docstring"""


				if os.path.isdir(self._tfds_path			):
					lowerCAmelCase												=  os.path.abspath(self._tfds_path			)
				elif os.path.isfile(self._tfds_path			):
					lowerCAmelCase												=  os.path.dirname(self._tfds_path			)
				else:
					raise ValueError('''--tfds_path is neither a directory nor a file. Please check path.'''			)

				lowerCAmelCase												=  os.path.abspath(self._datasets_directory			)

				self._logger.info(F"""Converting datasets from {abs_tfds_path} to {abs_datasets_path}"""			)

				lowerCAmelCase												=  []
				lowerCAmelCase												=  []
				lowerCAmelCase												=  {}

				if os.path.isdir(self._tfds_path			):
					lowerCAmelCase												=  os.listdir(__snake_case			)
				else:
					lowerCAmelCase												=  [os.path.basename(self._tfds_path			)]

				for f_name in file_names:
					self._logger.info(F"""Looking at file {f_name}"""			)
					lowerCAmelCase												=  os.path.join(__snake_case      ,__snake_case			)
					lowerCAmelCase												=  os.path.join(__snake_case      ,__snake_case			)

					if not os.path.isfile(__snake_case			) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
						self._logger.info('''Skipping file'''			)
						continue

					with open(__snake_case      ,encoding='''utf-8'''			) as f:
						lowerCAmelCase												=  f.readlines()

					lowerCAmelCase												=  []
					lowerCAmelCase												=  False
					lowerCAmelCase												=  False
					lowerCAmelCase												=  []
					for line in lines:
						lowerCAmelCase												=  line

						# Convert imports
						if "import tensorflow.compat.v2 as tf" in out_line:
							continue
						elif "@tfds.core" in out_line:
							continue
						elif "builder=self" in out_line:
							continue
						elif "import tensorflow_datasets.public_api as tfds" in out_line:
							lowerCAmelCase												=  """import datasets\n"""
						elif "import tensorflow" in out_line:
							# order is important here
							lowerCAmelCase												=  """"""
							continue
						elif "from absl import logging" in out_line:
							lowerCAmelCase												=  """from datasets import logging\n"""
						elif "getLogger" in out_line:
							lowerCAmelCase												=  out_line.replace('''getLogger'''      ,'''get_logger'''			)
						elif any(expression in out_line for expression in TO_HIGHLIGHT			):
							lowerCAmelCase												=  True
							lowerCAmelCase												=  list(filter(lambda SCREAMING_SNAKE_CASE_			: e in out_line      ,__snake_case			)			)
							out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(__snake_case			) + '''\n'''			)
							out_lines.append(__snake_case			)
							out_lines.append(__snake_case			)
							continue
						else:
							for pattern, replacement in TO_CONVERT:
								lowerCAmelCase												=  re.sub(__snake_case      ,__snake_case      ,__snake_case			)

                # Take care of saving utilities (to later move them together with main script)
						if "tensorflow_datasets" in out_line:
							lowerCAmelCase												=  re.match(r'''from\stensorflow_datasets.*import\s([^\.\r\n]+)'''      ,__snake_case			)
							tfds_imports.extend(imp.strip() for imp in match.group(1			).split(''','''			)			)
							lowerCAmelCase												=  """from . import """ + match.group(1			)

						# Check we have not forget anything
						if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
							raise ValueError(F"""Error converting {out_line.strip()}"""			)

						if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
							lowerCAmelCase												=  True
						out_lines.append(__snake_case			)

					if is_builder or "wmt" in f_name:
						# We create a new directory for each dataset
						lowerCAmelCase												=  f_name.replace('''.py'''      ,''''''			)
						lowerCAmelCase												=  os.path.join(__snake_case      ,__snake_case			)
						lowerCAmelCase												=  os.path.join(__snake_case      ,__snake_case			)
						os.makedirs(__snake_case      ,exist_ok=__snake_case			)
						self._logger.info(F"""Adding directory {output_dir}"""			)
						imports_to_builder_map.update({imp: output_dir for imp in tfds_imports}			)
					else:
						# Utilities will be moved at the end
						utils_files.append(__snake_case			)

					if needs_manual_update:
						with_manual_update.append(__snake_case			)

					with open(__snake_case      ,'''w'''      ,encoding='''utf-8'''			) as f:
						f.writelines(__snake_case			)
					self._logger.info(F"""Converted in {output_file}"""			)

				for utils_file in utils_files:
					try:
						lowerCAmelCase												=  os.path.basename(__snake_case			)
						lowerCAmelCase												=  imports_to_builder_map[f_name.replace('''.py'''      ,''''''			)]
						self._logger.info(F"""Moving {dest_folder} to {utils_file}"""			)
						shutil.copy(__snake_case      ,__snake_case			)
					except KeyError:
						self._logger.error(F"""Cannot find destination folder for {utils_file}. Please copy manually."""			)

				if with_manual_update:
					for file_path in with_manual_update:
						self._logger.warning(
						    F"""You need to manually update file {file_path} to remove configurations using 'TextEncoderConfig'."""			)






'''simple docstring'''


import subprocess
import sys

from transformers import BertConfig, BertModel, BertTokenizer, pipeline
from transformers.testing_utils import TestCasePlus, require_torch





class        lowercase__    (		_snake_case   ):





							'''simple docstring'''


							@require_torch
							def UpperCAmelCase_      (       self    ):
											# this test is a bit tricky since TRANSFORMERS_OFFLINE can only be changed before
											# `transformers` is loaded, and it's too late for inside pytest - so we are changing it
											# while running an external program

											# python one-liner segments

											# this must be loaded before socket.socket is monkey-patched
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= """
from transformers import BertConfig, BertModel, BertTokenizer, pipeline
        """

											_SCREAMING_SNAKE_CASE					:			Dict							= """
mname = \"hf-internal-testing/tiny-random-bert\"
BertConfig.from_pretrained(mname)
BertModel.from_pretrained(mname)
BertTokenizer.from_pretrained(mname)
pipe = pipeline(task=\"fill-mask\", model=mname)
print(\"success\")
        """

											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= """
import socket
def offline_socket(*args, **kwargs): raise RuntimeError(\"Offline mode is enabled, we shouldn't access internet\")
socket.socket = offline_socket
        """

											# Force fetching the files so that we can use the cache
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= """hf-internal-testing/tiny-random-bert"""
											BertConfig.from_pretrained(__snake_case    )
											BertModel.from_pretrained(__snake_case    )
											BertTokenizer.from_pretrained(__snake_case    )
											pipeline(task="""fill-mask"""       ,   model=__snake_case    )

											# baseline - just load from_pretrained with normal network
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= [sys.executable, """-c""", """\n""".join([load, run, mock]    )]

											# should succeed
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= self.get_env()
											# should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
											_SCREAMING_SNAKE_CASE					:			int							= """1"""
											_SCREAMING_SNAKE_CASE					:			str							= subprocess.run(__snake_case       ,   env=__snake_case       ,   check=__snake_case       ,   capture_output=__snake_case    )
											self.assertEqual(result.returncode       ,   0       ,   result.stderr    )
											self.assertIn("""success"""       ,   result.stdout.decode()    )






							@require_torch
							def UpperCAmelCase_      (       self    ):
											# python one-liner segments
											# this must be loaded before socket.socket is monkey-patched
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= """
from transformers import BertConfig, BertModel, BertTokenizer, pipeline
        """

											_SCREAMING_SNAKE_CASE					:			Any							= """
mname = \"hf-internal-testing/tiny-random-bert\"
BertConfig.from_pretrained(mname)
BertModel.from_pretrained(mname)
BertTokenizer.from_pretrained(mname)
pipe = pipeline(task=\"fill-mask\", model=mname)
print(\"success\")
        """

											_SCREAMING_SNAKE_CASE					:			List[Any]							= """
import socket
def offline_socket(*args, **kwargs): raise socket.error(\"Faking flaky internet\")
socket.socket = offline_socket
        """

											# Force fetching the files so that we can use the cache
											_SCREAMING_SNAKE_CASE					:			Tuple							= """hf-internal-testing/tiny-random-bert"""
											BertConfig.from_pretrained(__snake_case    )
											BertModel.from_pretrained(__snake_case    )
											BertTokenizer.from_pretrained(__snake_case    )
											pipeline(task="""fill-mask"""       ,   model=__snake_case    )

											# baseline - just load from_pretrained with normal network
											_SCREAMING_SNAKE_CASE					:			Tuple							= [sys.executable, """-c""", """\n""".join([load, run, mock]    )]

											# should succeed
											_SCREAMING_SNAKE_CASE					:			str							= self.get_env()
											_SCREAMING_SNAKE_CASE					:			Tuple							= subprocess.run(__snake_case       ,   env=__snake_case       ,   check=__snake_case       ,   capture_output=__snake_case    )
											self.assertEqual(result.returncode       ,   0       ,   result.stderr    )
											self.assertIn("""success"""       ,   result.stdout.decode()    )






							@require_torch
							def UpperCAmelCase_      (       self    ):
											# this test is a bit tricky since TRANSFORMERS_OFFLINE can only be changed before
											# `transformers` is loaded, and it's too late for inside pytest - so we are changing it
											# while running an external program

											# python one-liner segments

											# this must be loaded before socket.socket is monkey-patched
											_SCREAMING_SNAKE_CASE					:			Tuple							= """
from transformers import BertConfig, BertModel, BertTokenizer
        """

											_SCREAMING_SNAKE_CASE					:			str							= """
mname = \"hf-internal-testing/tiny-random-bert-sharded\"
BertConfig.from_pretrained(mname)
BertModel.from_pretrained(mname)
print(\"success\")
        """

											_SCREAMING_SNAKE_CASE					:			List[Any]							= """
import socket
def offline_socket(*args, **kwargs): raise ValueError(\"Offline mode is enabled\")
socket.socket = offline_socket
        """

											# baseline - just load from_pretrained with normal network
											_SCREAMING_SNAKE_CASE					:			List[str]							= [sys.executable, """-c""", """\n""".join([load, run]    )]

											# should succeed
											_SCREAMING_SNAKE_CASE					:			List[str]							= self.get_env()
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= subprocess.run(__snake_case       ,   env=__snake_case       ,   check=__snake_case       ,   capture_output=__snake_case    )
											self.assertEqual(result.returncode       ,   0       ,   result.stderr    )
											self.assertIn("""success"""       ,   result.stdout.decode()    )

											# next emulate no network
											_SCREAMING_SNAKE_CASE					:			List[str]							= [sys.executable, """-c""", """\n""".join([load, mock, run]    )]

											# Doesn't fail anymore since the model is in the cache due to other tests, so commenting this.
											# env["TRANSFORMERS_OFFLINE"] = "0"
											# result = subprocess.run(cmd, env=env, check=False, capture_output=True)
											# self.assertEqual(result.returncode, 1, result.stderr)

											# should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
											_SCREAMING_SNAKE_CASE					:			Dict							= """1"""
											_SCREAMING_SNAKE_CASE					:			int							= subprocess.run(__snake_case       ,   env=__snake_case       ,   check=__snake_case       ,   capture_output=__snake_case    )
											self.assertEqual(result.returncode       ,   0       ,   result.stderr    )
											self.assertIn("""success"""       ,   result.stdout.decode()    )






							@require_torch
							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= """
from transformers import pipeline
        """
											_SCREAMING_SNAKE_CASE					:			int							= """
mname = \"hf-internal-testing/tiny-random-bert\"
pipe = pipeline(model=mname)
        """

											_SCREAMING_SNAKE_CASE					:			Any							= """
import socket
def offline_socket(*args, **kwargs): raise socket.error(\"Offline mode is enabled\")
socket.socket = offline_socket
        """
											_SCREAMING_SNAKE_CASE					:			Dict							= self.get_env()
											_SCREAMING_SNAKE_CASE					:			Any							= """1"""
											_SCREAMING_SNAKE_CASE					:			str							= [sys.executable, """-c""", """\n""".join([load, mock, run]    )]
											_SCREAMING_SNAKE_CASE					:			List[Any]							= subprocess.run(__snake_case       ,   env=__snake_case       ,   check=__snake_case       ,   capture_output=__snake_case    )
											self.assertEqual(result.returncode       ,   1       ,   result.stderr    )
											self.assertIn(
											    """You cannot infer task automatically within `pipeline` when using offline mode"""       ,   result.stderr.decode().replace("""\n"""       ,   """"""    )       ,   )






							@require_torch
							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= """
from transformers import AutoModel
        """
											_SCREAMING_SNAKE_CASE					:			Tuple							= """
mname = \"hf-internal-testing/test_dynamic_model\"
AutoModel.from_pretrained(mname, trust_remote_code=True)
print(\"success\")
        """

											# baseline - just load from_pretrained with normal network
											_SCREAMING_SNAKE_CASE					:			List[str]							= [sys.executable, """-c""", """\n""".join([load, run]    )]

											# should succeed
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= self.get_env()
											_SCREAMING_SNAKE_CASE					:			List[str]							= subprocess.run(__snake_case       ,   env=__snake_case       ,   check=__snake_case       ,   capture_output=__snake_case    )
											self.assertEqual(result.returncode       ,   0       ,   result.stderr    )
											self.assertIn("""success"""       ,   result.stdout.decode()    )

											# should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= """1"""
											_SCREAMING_SNAKE_CASE					:			Any							= subprocess.run(__snake_case       ,   env=__snake_case       ,   check=__snake_case       ,   capture_output=__snake_case    )
											self.assertEqual(result.returncode       ,   0       ,   result.stderr    )
											self.assertIn("""success"""       ,   result.stdout.decode()    )




