






"""simple docstring"""




import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


UpperCAmelCase_       :							Optional[Any]             =				datasets.utils.logging.get_logger(__name__)


@dataclass
class 	__UpperCAmelCase		(				datasets.BuilderConfig	):


	'''simple docstring'''





	lowercase		:	Optional[datasets.Features]							     =				None
	lowercase		:	str							     =				"utf-8"
	lowercase		:	Optional[str]							     =				None
	lowercase		:	Optional[str]							     =				None
	lowercase		:	bool							     =				True  # deprecated
	lowercase		:	Optional[int]							     =				None  # deprecated
	lowercase		:	int							     =				10 << 20  # 10MB
	lowercase		:	Optional[bool]							     =				None




class 	__UpperCAmelCase		(				datasets.ArrowBasedBuilder	):


	'''simple docstring'''





	lowercase		:	List[Any]							     =				JsonConfig


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		if self.config.block_size is not None:
			logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''					)
			_SCREAMING_SNAKE_CASE						=self.config.block_size
		if self.config.use_threads is not True:
			logger.warning(
			    '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''					)
		if self.config.newlines_in_values is not None:
			raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''					)
		return datasets.DatasetInfo(features=self.config.features					)


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		if not self.config.data_files:
			raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""					)
		_SCREAMING_SNAKE_CASE						=dl_manager.download_and_extract(self.config.data_files					)
		if isinstance(__lowerCAmelCase      ,      (str, list, tuple)					):
			_SCREAMING_SNAKE_CASE						=data_files
			if isinstance(__lowerCAmelCase      ,      __lowerCAmelCase					):
				_SCREAMING_SNAKE_CASE						=[files]
			_SCREAMING_SNAKE_CASE						=[dl_manager.iter_files(__lowerCAmelCase					) for file in files]
			return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,      gen_kwargs={'''files''': files}					)]
		_SCREAMING_SNAKE_CASE						=[]
		for split_name, files in data_files.items():
			if isinstance(__lowerCAmelCase      ,      __lowerCAmelCase					):
				_SCREAMING_SNAKE_CASE						=[files]
			_SCREAMING_SNAKE_CASE						=[dl_manager.iter_files(__lowerCAmelCase					) for file in files]
			splits.append(datasets.SplitGenerator(name=__lowerCAmelCase      ,      gen_kwargs={'''files''': files}					)					)
		return splits


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		if self.config.features is not None:
			# adding missing columns
			for column_name in set(self.config.features					) - set(pa_table.column_names					):
				_SCREAMING_SNAKE_CASE						=self.config.features.arrow_schema.field(__lowerCAmelCase					).type
				_SCREAMING_SNAKE_CASE						=pa_table.append_column(__lowerCAmelCase      ,      pa.array([None] * len(__lowerCAmelCase					)      ,      type=__lowerCAmelCase					)					)
			# more expensive cast to support nested structures with keys in a different order
			# allows str <-> int/float or str to Audio for example
			_SCREAMING_SNAKE_CASE						=table_cast(__lowerCAmelCase      ,      self.config.features.arrow_schema					)
		return pa_table







	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		for file_idx, file in enumerate(itertools.chain.from_iterable(__lowerCAmelCase					)					):
			# If the file is one json object and if we need to look at the list of items in one specific field
			if self.config.field is not None:
				with open(__lowerCAmelCase      ,      encoding=self.config.encoding      ,      errors=self.config.encoding_errors					) as f:
					_SCREAMING_SNAKE_CASE						=json.load(__lowerCAmelCase					)

				# We keep only the field we are interested in
				_SCREAMING_SNAKE_CASE						=dataset[self.config.field]

				# We accept two format: a list of dicts or a dict of lists
				if isinstance(__lowerCAmelCase      ,      (list, tuple)					):
					_SCREAMING_SNAKE_CASE						=set().union(*[row.keys() for row in dataset]					)
					_SCREAMING_SNAKE_CASE						={col: [row.get(__lowerCAmelCase					) for row in dataset] for col in keys}
				else:
					_SCREAMING_SNAKE_CASE						=dataset
				_SCREAMING_SNAKE_CASE						=pa.Table.from_pydict(__lowerCAmelCase					)
				yield file_idx, self._cast_table(__lowerCAmelCase					)

			# If the file has one json object per line
			else:
				with open(__lowerCAmelCase      ,      '''rb'''					) as f:
					_SCREAMING_SNAKE_CASE						=0
					# Use block_size equal to the chunk size divided by 32 to leverage multithreading
					# Set a default minimum value of 16kB if the chunk size is really small
					_SCREAMING_SNAKE_CASE						=max(self.config.chunksize // 3_2      ,      1_6 << 1_0					)
					_SCREAMING_SNAKE_CASE						=(
					    self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
					)
					while True:
						_SCREAMING_SNAKE_CASE						=f.read(self.config.chunksize					)
						if not batch:
							break
						# Finish current line
						try:
							batch += f.readline()
						except (AttributeError, io.UnsupportedOperation):
							batch += readline(__lowerCAmelCase					)
						# PyArrow only accepts utf-8 encoded bytes
						if self.config.encoding != "utf-8":
							_SCREAMING_SNAKE_CASE						=batch.decode(self.config.encoding      ,      errors=__lowerCAmelCase					).encode('''utf-8'''					)
						try:
							while True:
								try:
									_SCREAMING_SNAKE_CASE						=paj.read_json(
									    io.BytesIO(__lowerCAmelCase					)      ,      read_options=paj.ReadOptions(block_size=__lowerCAmelCase					)					)
									break
								except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
									if (
									    isinstance(__lowerCAmelCase      ,      pa.ArrowInvalid					)
									    and "straddling" not in str(__lowerCAmelCase					)
									    or block_size > len(__lowerCAmelCase					)
									):
										raise
									else:
										# Increase the block size in case it was too small.
										# The block size will be reset for the next file.
										logger.debug(
										    f"""Batch of {len(__lowerCAmelCase					)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""					)
										block_size *= 2
						except pa.ArrowInvalid as e:
							try:
								with open(
								    __lowerCAmelCase      ,      encoding=self.config.encoding      ,      errors=self.config.encoding_errors					) as f:
									_SCREAMING_SNAKE_CASE						=json.load(__lowerCAmelCase					)
							except json.JSONDecodeError:
								logger.error(f"""Failed to read file \'{file}\' with error {type(__lowerCAmelCase					)}: {e}"""					)
								raise e
							# If possible, parse the file as a list of json objects and exit the loop
							if isinstance(__lowerCAmelCase      ,      __lowerCAmelCase					):  # list is the only sequence type supported in JSON
								try:
									_SCREAMING_SNAKE_CASE						=set().union(*[row.keys() for row in dataset]					)
									_SCREAMING_SNAKE_CASE						={col: [row.get(__lowerCAmelCase					) for row in dataset] for col in keys}
									_SCREAMING_SNAKE_CASE						=pa.Table.from_pydict(__lowerCAmelCase					)
								except (pa.ArrowInvalid, AttributeError) as e:
									logger.error(f"""Failed to read file \'{file}\' with error {type(__lowerCAmelCase					)}: {e}"""					)
									raise ValueError(f"""Not able to read records in the JSON file at {file}."""					) from None
								yield file_idx, self._cast_table(__lowerCAmelCase					)
								break
							else:
								logger.error(f"""Failed to read file \'{file}\' with error {type(__lowerCAmelCase					)}: {e}"""					)
								raise ValueError(
								    f"""Not able to read records in the JSON file at {file}. """
								    f"""You should probably indicate the field of the JSON file containing your records. """
								    f"""This JSON file contain the following fields: {str(list(dataset.keys()					)					)}. """
								    f"""Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. """					) from None
								  # Uncomment for debugging (will print the Arrow table size and elements)
								  # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
								  # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
						yield (file_idx, batch_idx), self._cast_table(__lowerCAmelCase					)
						batch_idx += 1









import collections
import os
import re
from pathlib import Path


UpperCAmelCase_		 =					"""src/transformers"""


# Matches is_xxx_available()
UpperCAmelCase_		 =					re.compile(r"""is\_([a-z_]*)_available()""")
# Catches a one-line _import_struct = {xxx}
UpperCAmelCase_		 =					re.compile(r"""^_import_structure\s+=\s+\{([^\}]+)\}""")
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
UpperCAmelCase_		 =					re.compile(r"""\s+\"\S*\":\s+\[([^\]]*)\]""")
# Catches a line if not is_foo_available
UpperCAmelCase_		 =					re.compile(r"""^\s*if\s+not\s+is\_[a-z_]*\_available\(\)""")
# Catches a line _import_struct["bla"].append("foo")
UpperCAmelCase_		 =					re.compile(r"""^\s*_import_structure\[\"\S*\"\]\.append\(\"(\S*)\"\)""")
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
UpperCAmelCase_		 =					re.compile(r"""^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]""")
# Catches a line with an object between quotes and a comma:     "MyModel",
UpperCAmelCase_		 =					re.compile(r"""^\s+\"([^\"]+)\",""")
# Catches a line with objects between brackets only:    ["foo", "bar"],
UpperCAmelCase_		 =					re.compile(r"""^\s+\[([^\]]+)\]""")
# Catches a line with from foo import bar, bla, boo
UpperCAmelCase_		 =					re.compile(r"""\s+from\s+\S*\s+import\s+([^\(\s].*)\n""")
# Catches a line with try:
UpperCAmelCase_		 =					re.compile(r"""^\s*try:""")
# Catches a line with else:
UpperCAmelCase_		 =					re.compile(r"""^\s*else:""")

def 	SCREAMING_SNAKE_CASE_							(	_snake_case	:Optional[int]				)  ->     Any:
   if _re_test_backend.search(_snake_case				) is None:
      return None
   _A      =       [b[0] for b in _re_backend.findall(_snake_case				)]
   backends.sort()
   return "_and_".join(_snake_case				)

def 	SCREAMING_SNAKE_CASE_							(	_snake_case	:Any				)  ->     Any:
   with open(_snake_case	,   '''r'''	,   encoding='''utf-8'''	,   newline='''\n'''				) as f:
      _A      =       f.readlines()

   _A      =       0
   while line_index < len(_snake_case				) and not lines[line_index].startswith('''_import_structure = {'''				):
      line_index += 1

   # If this is a traditional init, just return.
   if line_index >= len(_snake_case				):
      return None

   # First grab the objects without a specific backend in _import_structure
   _A      =       []
   while not lines[line_index].startswith('''if TYPE_CHECKING'''				) and find_backend(lines[line_index]				) is None:
      _A      =       lines[line_index]
      # If we have everything on a single line, let's deal with it.
      if _re_one_line_import_struct.search(_snake_case				):
         _A      =       _re_one_line_import_struct.search(_snake_case				).groups()[0]
         _A      =       re.findall(r'''\[([^\]]+)\]'''	,   _snake_case				)
         for imp in imports:
            objects.extend([obj[1:-1] for obj in imp.split(''', '''				)]				)
         line_index += 1
         continue
      _A      =       _re_import_struct_key_value.search(_snake_case				)
      if single_line_import_search is not None:
         _A      =       [obj[1:-1] for obj in single_line_import_search.groups()[0].split(''', '''				) if len(_snake_case				) > 0]
         objects.extend(_snake_case				)
      elif line.startswith(''' ''' * 8 + '''"'''				):
         objects.append(line[9:-3]				)
      line_index += 1

   _A      =       {'''none''': objects}
   # Let's continue with backend-specific objects in _import_structure
   while not lines[line_index].startswith('''if TYPE_CHECKING'''				):
      # If the line is an if not is_backend_available, we grab all objects associated.
      _A      =       find_backend(lines[line_index]				)
      # Check if the backend declaration is inside a try block:
      if _re_try.search(lines[line_index - 1]				) is None:
         _A      =       None

      if backend is not None:
         line_index += 1

         # Scroll until we hit the else block of try-except-else
         while _re_else.search(lines[line_index]				) is None:
            line_index += 1

         line_index += 1

         _A      =       []
         # Until we unindent, add backend objects to the list
         while len(lines[line_index]				) <= 1 or lines[line_index].startswith(''' ''' * 4				):
            _A      =       lines[line_index]
            if _re_import_struct_add_one.search(_snake_case				) is not None:
               objects.append(_re_import_struct_add_one.search(_snake_case				).groups()[0]				)
            elif _re_import_struct_add_many.search(_snake_case				) is not None:
               _A      =       _re_import_struct_add_many.search(_snake_case				).groups()[0].split(''', '''				)
               _A      =       [obj[1:-1] for obj in imports if len(_snake_case				) > 0]
               objects.extend(_snake_case				)
            elif _re_between_brackets.search(_snake_case				) is not None:
               _A      =       _re_between_brackets.search(_snake_case				).groups()[0].split(''', '''				)
               _A      =       [obj[1:-1] for obj in imports if len(_snake_case				) > 0]
               objects.extend(_snake_case				)
            elif _re_quote_object.search(_snake_case				) is not None:
               objects.append(_re_quote_object.search(_snake_case				).groups()[0]				)
            elif line.startswith(''' ''' * 8 + '''"'''				):
               objects.append(line[9:-3]				)
            elif line.startswith(''' ''' * 12 + '''"'''				):
               objects.append(line[13:-3]				)
            line_index += 1

         _A      =       objects
      else:
         line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
   _A      =       []
   while (
       line_index < len(_snake_case				)
       and find_backend(lines[line_index]				) is None
       and not lines[line_index].startswith('''else'''				)
   ):
      _A      =       lines[line_index]
      _A      =       _re_import.search(_snake_case				)
      if single_line_import_search is not None:
         objects.extend(single_line_import_search.groups()[0].split(''', '''				)				)
      elif line.startswith(''' ''' * 8				):
         objects.append(line[8:-2]				)
      line_index += 1

   _A      =       {'''none''': objects}
   # Let's continue with backend-specific objects
   while line_index < len(_snake_case				):
      # If the line is an if is_backend_available, we grab all objects associated.
      _A      =       find_backend(lines[line_index]				)
      # Check if the backend declaration is inside a try block:
      if _re_try.search(lines[line_index - 1]				) is None:
         _A      =       None

      if backend is not None:
         line_index += 1

         # Scroll until we hit the else block of try-except-else
         while _re_else.search(lines[line_index]				) is None:
            line_index += 1

         line_index += 1

         _A      =       []
         # Until we unindent, add backend objects to the list
         while len(lines[line_index]				) <= 1 or lines[line_index].startswith(''' ''' * 8				):
            _A      =       lines[line_index]
            _A      =       _re_import.search(_snake_case				)
            if single_line_import_search is not None:
               objects.extend(single_line_import_search.groups()[0].split(''', '''				)				)
            elif line.startswith(''' ''' * 12				):
               objects.append(line[12:-2]				)
            line_index += 1

         _A      =       objects
      else:
         line_index += 1

   return import_dict_objects, type_hint_objects

def 	SCREAMING_SNAKE_CASE_							(	_snake_case	:List[str]	,   _snake_case	:Dict				)  ->     Any:

   def find_duplicates(_snake_case	:Any				):
      return [k for k, v in collections.Counter(_snake_case				).items() if v > 1]

   if list(import_dict_objects.keys()				) != list(type_hint_objects.keys()				):
      return ["Both sides of the init do not have the same backends!"]

   _A      =       []
   for key in import_dict_objects.keys():
      _A      =       find_duplicates(import_dict_objects[key]				)
      if duplicate_imports:
         errors.append(F'''Duplicate _import_structure definitions for: {duplicate_imports}'''				)
      _A      =       find_duplicates(type_hint_objects[key]				)
      if duplicate_type_hints:
         errors.append(F'''Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'''				)

      if sorted(set(import_dict_objects[key]				)				) != sorted(set(type_hint_objects[key]				)				):
         _A      =       '''base imports''' if key == '''none''' else F'''{key} backend'''
         errors.append(F'''Differences for {name}:'''				)
         for a in type_hint_objects[key]:
            if a not in import_dict_objects[key]:
               errors.append(F'''  {a} in TYPE_HINT but not in _import_structure.'''				)
         for a in import_dict_objects[key]:
            if a not in type_hint_objects[key]:
               errors.append(F'''  {a} in _import_structure but not in TYPE_HINT.'''				)
   return errors

def 	SCREAMING_SNAKE_CASE_							(	)  ->     int:
   _A      =       []
   for root, _, files in os.walk(_snake_case				):
      if "__init__.py" in files:
         _A      =       os.path.join(_snake_case	,   '''__init__.py'''				)
         _A      =       parse_init(_snake_case				)
         if objects is not None:
            _A      =       analyze_results(*_snake_case				)
            if len(_snake_case				) > 0:
               _A      =       F'''Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'''
               failures.append('''\n'''.join(_snake_case				)				)
   if len(_snake_case				) > 0:
      raise ValueError('''\n\n'''.join(_snake_case				)				)

def 	SCREAMING_SNAKE_CASE_							(	)  ->     Optional[int]:
   _A      =       []
   for path, directories, files in os.walk(_snake_case				):
      for folder in directories:
         # Ignore private modules
         if folder.startswith('''_'''				):
            directories.remove(_snake_case				)
            continue
         # Ignore leftovers from branches (empty folders apart from pycache)
         if len(list((Path(_snake_case				) / folder).glob('''*.py'''				)				)				) == 0:
            continue
         _A      =       str((Path(_snake_case				) / folder).relative_to(_snake_case				)				)
         _A      =       short_path.replace(os.path.sep	,   '''.'''				)
         submodules.append(_snake_case				)
      for fname in files:
         if fname == "__init__.py":
            continue
         _A      =       str((Path(_snake_case				) / fname).relative_to(_snake_case				)				)
         _A      =       short_path.replace('''.py'''	,   ''''''				).replace(os.path.sep	,   '''.'''				)
         if len(submodule.split('''.'''				)				) == 1:
            submodules.append(_snake_case				)
   return submodules


UpperCAmelCase_		 =					[
    """convert_pytorch_checkpoint_to_tf2""",
    """modeling_flax_pytorch_utils""",
    """models.esm.openfold_utils""",
]

def 	SCREAMING_SNAKE_CASE_							(	)  ->     List[str]:
   # This is to make sure the transformers module imported is the one in the repo.
   from transformers.utils import direct_transformers_import

   _A      =       direct_transformers_import(_snake_case				)

   _A      =       set(transformers._import_structure.keys()				)
   # This contains all the base keys of the _import_structure object defined in the init, but if the user is missing
   # some optional dependencies, they may not have all of them. Thus we read the init to read all additions and
   # (potentiall re-) add them.
   with open(os.path.join(_snake_case	,   '''__init__.py'''				)	,   '''r'''				) as f:
      _A      =       f.read()
   import_structure_keys.update(set(re.findall(r'''import_structure\[\"([^\"]*)\"\]'''	,   _snake_case				)				)				)

   _A      =       [
       module
       for module in get_transformers_submodules()
       if module not in IGNORE_SUBMODULES and module not in import_structure_keys
   ]

   if len(_snake_case				) > 0:
      _A      =       '''\n'''.join(F'''- {module}''' for module in module_not_registered				)
      raise ValueError(
          '''The following submodules are not properly registed in the main init of Transformers:\n'''
          F'''{list_of_modules}\n'''
          '''Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'''				)


if __name__ == "__main__":
   check_all_inits()
   check_submodules()



