




"""simple docstring"""




import collections
import importlib.util
import os
import re
from pathlib import Path


a_        =       'src/transformers'


# Matches is_xxx_available()
a_        =       re.compile(r'is\_([a-z_]*)_available()')
# Catches a one-line _import_struct = {xxx}
a_        =       re.compile(r'^_import_structure\s+=\s+\{([^\}]+)\}')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
a_        =       re.compile(r'\s+"\S*":\s+\[([^\]]*)\]')
# Catches a line if not is_foo_available
a_        =       re.compile(r'^\s*if\s+not\s+is\_[a-z_]*\_available\(\)')
# Catches a line _import_struct["bla"].append("foo")
a_        =       re.compile(r'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
a_        =       re.compile(r'^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]')
# Catches a line with an object between quotes and a comma:     "MyModel",
a_        =       re.compile('^\s+"([^"]+)",')
# Catches a line with objects between brackets only:    ["foo", "bar"],
a_        =       re.compile('^\s+\[([^\]]+)\]')
# Catches a line with from foo import bar, bla, boo
a_        =       re.compile(r'\s+from\s+\S*\s+import\s+([^\(\s].*)\n')
# Catches a line with try:
a_        =       re.compile(r'^\s*try:')
# Catches a line with else:
a_        =       re.compile(r'^\s*else:')







def      __UpperCAmelCase     ( __UpperCamelCase	):
			if _re_test_backend.search(__UpperCamelCase	) is None:
						return None
			__lowercase			:  List[str]								=       [b[0] for b in _re_backend.findall(__UpperCamelCase	)]
			backends.sort()
			return "_and_".join(__UpperCamelCase	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			with open(__UpperCamelCase	,				'''r'''	,				encoding='''utf-8'''	,				newline='''\n'''	) as f:
						__lowercase			:  Tuple								=       f.readlines()

			__lowercase			:  str								=       0
			while line_index < len(__UpperCamelCase	) and not lines[line_index].startswith('''_import_structure = {'''	):
						line_index += 1

			# If this is a traditional init, just return.
			if line_index >= len(__UpperCamelCase	):
						return None

			# First grab the objects without a specific backend in _import_structure
			__lowercase			:  List[str]								=       []
			while not lines[line_index].startswith('''if TYPE_CHECKING'''	) and find_backend(lines[line_index]	) is None:
						__lowercase			:  List[Any]								=       lines[line_index]
						# If we have everything on a single line, let's deal with it.
						if _re_one_line_import_struct.search(__UpperCamelCase	):
									__lowercase			:  int								=       _re_one_line_import_struct.search(__UpperCamelCase	).groups()[0]
									__lowercase			:  Union[str, Any]								=       re.findall('''\[([^\]]+)\]'''	,				__UpperCamelCase	)
									for imp in imports:
												objects.extend([obj[1:-1] for obj in imp.split(''', '''	)]	)
									line_index += 1
									continue
						__lowercase			:  Tuple								=       _re_import_struct_key_value.search(__UpperCamelCase	)
						if single_line_import_search is not None:
									__lowercase			:  Tuple								=       [obj[1:-1] for obj in single_line_import_search.groups()[0].split(''', '''	) if len(__UpperCamelCase	) > 0]
									objects.extend(__UpperCamelCase	)
						elif line.startswith(''' ''' * 8 + '''"'''	):
									objects.append(line[9:-3]	)
						line_index += 1

			__lowercase			:  Optional[Any]								=       {'''none''': objects}
			# Let's continue with backend-specific objects in _import_structure
			while not lines[line_index].startswith('''if TYPE_CHECKING'''	):
						# If the line is an if not is_backend_available, we grab all objects associated.
						__lowercase			:  Dict								=       find_backend(lines[line_index]	)
						# Check if the backend declaration is inside a try block:
						if _re_try.search(lines[line_index - 1]	) is None:
									__lowercase			:  str								=       None

						if backend is not None:
									line_index += 1

									# Scroll until we hit the else block of try-except-else
									while _re_else.search(lines[line_index]	) is None:
												line_index += 1

									line_index += 1

									__lowercase			:  Optional[Any]								=       []
									# Until we unindent, add backend objects to the list
									while len(lines[line_index]	) <= 1 or lines[line_index].startswith(''' ''' * 4	):
												__lowercase			:  Union[str, Any]								=       lines[line_index]
												if _re_import_struct_add_one.search(__UpperCamelCase	) is not None:
															objects.append(_re_import_struct_add_one.search(__UpperCamelCase	).groups()[0]	)
												elif _re_import_struct_add_many.search(__UpperCamelCase	) is not None:
															__lowercase			:  Optional[Any]								=       _re_import_struct_add_many.search(__UpperCamelCase	).groups()[0].split(''', '''	)
															__lowercase			:  Tuple								=       [obj[1:-1] for obj in imports if len(__UpperCamelCase	) > 0]
															objects.extend(__UpperCamelCase	)
												elif _re_between_brackets.search(__UpperCamelCase	) is not None:
															__lowercase			:  int								=       _re_between_brackets.search(__UpperCamelCase	).groups()[0].split(''', '''	)
															__lowercase			:  int								=       [obj[1:-1] for obj in imports if len(__UpperCamelCase	) > 0]
															objects.extend(__UpperCamelCase	)
												elif _re_quote_object.search(__UpperCamelCase	) is not None:
															objects.append(_re_quote_object.search(__UpperCamelCase	).groups()[0]	)
												elif line.startswith(''' ''' * 8 + '''"'''	):
															objects.append(line[9:-3]	)
												elif line.startswith(''' ''' * 12 + '''"'''	):
															objects.append(line[13:-3]	)
												line_index += 1

									__lowercase			:  str								=       objects
						else:
									line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
			__lowercase			:  Union[str, Any]								=       []
			while (
			    line_index < len(__UpperCamelCase	)
			    and find_backend(lines[line_index]	) is None
			    and not lines[line_index].startswith('''else'''	)
			):
						__lowercase			:  List[str]								=       lines[line_index]
						__lowercase			:  Optional[Any]								=       _re_import.search(__UpperCamelCase	)
						if single_line_import_search is not None:
									objects.extend(single_line_import_search.groups()[0].split(''', '''	)	)
						elif line.startswith(''' ''' * 8	):
									objects.append(line[8:-2]	)
						line_index += 1

			__lowercase			:  Tuple								=       {'''none''': objects}
			# Let's continue with backend-specific objects
			while line_index < len(__UpperCamelCase	):
						# If the line is an if is_backend_available, we grab all objects associated.
						__lowercase			:  List[Any]								=       find_backend(lines[line_index]	)
						# Check if the backend declaration is inside a try block:
						if _re_try.search(lines[line_index - 1]	) is None:
									__lowercase			:  Dict								=       None

						if backend is not None:
									line_index += 1

									# Scroll until we hit the else block of try-except-else
									while _re_else.search(lines[line_index]	) is None:
												line_index += 1

									line_index += 1

									__lowercase			:  Optional[int]								=       []
									# Until we unindent, add backend objects to the list
									while len(lines[line_index]	) <= 1 or lines[line_index].startswith(''' ''' * 8	):
												__lowercase			:  Optional[Any]								=       lines[line_index]
												__lowercase			:  Optional[int]								=       _re_import.search(__UpperCamelCase	)
												if single_line_import_search is not None:
															objects.extend(single_line_import_search.groups()[0].split(''', '''	)	)
												elif line.startswith(''' ''' * 12	):
															objects.append(line[12:-2]	)
												line_index += 1

									__lowercase			:  List[Any]								=       objects
						else:
									line_index += 1

			return import_dict_objects, type_hint_objects







def      __UpperCAmelCase     ( __UpperCamelCase	,				__UpperCamelCase	):

			def find_duplicates(__UpperCamelCase	):
						return [k for k, v in collections.Counter(__UpperCamelCase	).items() if v > 1]

			if list(import_dict_objects.keys()	) != list(type_hint_objects.keys()	):
						return ["Both sides of the init do not have the same backends!"]

			__lowercase			:  List[str]								=       []
			for key in import_dict_objects.keys():
						__lowercase			:  Optional[int]								=       find_duplicates(import_dict_objects[key]	)
						if duplicate_imports:
									errors.append(f"""Duplicate _import_structure definitions for: {duplicate_imports}"""	)
						__lowercase			:  List[Any]								=       find_duplicates(type_hint_objects[key]	)
						if duplicate_type_hints:
									errors.append(f"""Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}"""	)

						if sorted(set(import_dict_objects[key]	)	) != sorted(set(type_hint_objects[key]	)	):
									__lowercase			:  List[Any]								=       '''base imports''' if key == '''none''' else f"""{key} backend"""
									errors.append(f"""Differences for {name}:"""	)
									for a in type_hint_objects[key]:
												if a not in import_dict_objects[key]:
															errors.append(f"""  {a} in TYPE_HINT but not in _import_structure."""	)
									for a in import_dict_objects[key]:
												if a not in type_hint_objects[key]:
															errors.append(f"""  {a} in _import_structure but not in TYPE_HINT."""	)
			return errors







def      __UpperCAmelCase     ( ):
			__lowercase			:  Tuple								=       []
			for root, _, files in os.walk(__UpperCamelCase	):
						if "__init__.py" in files:
									__lowercase			:  Optional[int]								=       os.path.join(__UpperCamelCase	,				'''__init__.py'''	)
									__lowercase			:  Dict								=       parse_init(__UpperCamelCase	)
									if objects is not None:
												__lowercase			:  Dict								=       analyze_results(*__UpperCamelCase	)
												if len(__UpperCamelCase	) > 0:
															__lowercase			:  str								=       f"""Problem in {fname}, both halves do not define the same objects.\n{errors[0]}"""
															failures.append('''\n'''.join(__UpperCamelCase	)	)
			if len(__UpperCamelCase	) > 0:
						raise ValueError('''\n\n'''.join(__UpperCamelCase	)	)







def      __UpperCAmelCase     ( ):
			__lowercase			:  int								=       []
			for path, directories, files in os.walk(__UpperCamelCase	):
						for folder in directories:
									# Ignore private modules
									if folder.startswith('''_'''	):
												directories.remove(__UpperCamelCase	)
												continue
									# Ignore leftovers from branches (empty folders apart from pycache)
									if len(list((Path(__UpperCamelCase	) / folder).glob('''*.py'''	)	)	) == 0:
												continue
									__lowercase			:  Tuple								=       str((Path(__UpperCamelCase	) / folder).relative_to(__UpperCamelCase	)	)
									__lowercase			:  List[Any]								=       short_path.replace(os.path.sep	,				'''.'''	)
									submodules.append(__UpperCamelCase	)
						for fname in files:
									if fname == "__init__.py":
												continue
									__lowercase			:  Optional[Any]								=       str((Path(__UpperCamelCase	) / fname).relative_to(__UpperCamelCase	)	)
									__lowercase			:  List[str]								=       short_path.replace('''.py'''	,				''''''	).replace(os.path.sep	,				'''.'''	)
									if len(submodule.split('''.'''	)	) == 1:
												submodules.append(__UpperCamelCase	)
			return submodules


a_        =       [
    'convert_pytorch_checkpoint_to_tf2',
    'modeling_flax_pytorch_utils',
]







def      __UpperCAmelCase     ( ):
			# This is to make sure the transformers module imported is the one in the repo.
			__lowercase			:  str								=       importlib.util.spec_from_file_location(
			    '''transformers'''	,				os.path.join(__UpperCamelCase	,				'''__init__.py'''	)	,				submodule_search_locations=[PATH_TO_TRANSFORMERS]	,				)
			__lowercase			:  List[Any]								=       spec.loader.load_module()

			__lowercase			:  int								=       [
			    module
			    for module in get_transformers_submodules()
			    if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
			]
			if len(__UpperCamelCase	) > 0:
						__lowercase			:  Optional[Any]								=       '''\n'''.join(f"""- {module}""" for module in module_not_registered	)
						raise ValueError(
						    '''The following submodules are not properly registered in the main init of Transformers:\n'''
						    f"""{list_of_modules}\n"""
						    '''Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'''	)


if __name__ == "__main__":
			check_all_inits()
			check_submodules()









from typing import Dict, List, Optional, Union

import numpy as np

from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from .utils import PaddingStrategy, TensorType, is_tf_tensor, is_torch_tensor, logging, to_numpy


UpperCAmelCase_		:					Dict          =  logging.get_logger(__name__)




class 		__A   (      UpperCamelCase__      ):

						def __init__(		self :List[str]     ,			__snake_case :int     ,			__snake_case :int     ,			__snake_case :float     ,			**__snake_case :Optional[Any]			):
										'''simple docstring'''

										__magic_name__						:						List[Any]										=feature_size
										__magic_name__						:						Union[str, Any]										=sampling_rate
										__magic_name__						:						List[Any]										=padding_value

										__magic_name__						:						List[str]										=kwargs.pop("""padding_side"""     ,			"""right"""			)
										__magic_name__						:						Tuple										=kwargs.pop("""return_attention_mask"""     ,			__snake_case			)

										super().__init__(**__snake_case			)

						def      A__    (		self :Any     ,			__snake_case :Union[
						        BatchFeature,
						        List[BatchFeature],
						        Dict[str, BatchFeature],
						        Dict[str, List[BatchFeature]],
						        List[Dict[str, BatchFeature]],
						    ]     ,			__snake_case :Union[bool, str, PaddingStrategy] = True     ,			__snake_case :Optional[int] = None     ,			__snake_case :bool = False     ,			__snake_case :Optional[int] = None     ,			__snake_case :Optional[bool] = None     ,			__snake_case :Optional[Union[str, TensorType]] = None     ,			):
										'''simple docstring'''

										if isinstance(__snake_case     ,			(list, tuple)			) and isinstance(processed_features[0]     ,			(dict, BatchFeature)			):
														__magic_name__						:						Union[str, Any]										={
														    key: [example[key] for example in processed_features] for key in processed_features[0].keys()
														}

										# The model's main input name, usually `input_values`, has be passed for padding
										if self.model_input_names[0] not in processed_features:
														raise ValueError(
														    """You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`"""
														    f" to this method that includes {self.model_input_names[0]}, but you provided"
														    f" {list(processed_features.keys()			)}"			)

										__magic_name__						:						int										=processed_features[self.model_input_names[0]]
										__magic_name__						:						Union[str, Any]										=(
										    return_attention_mask if return_attention_mask is not None else self.return_attention_mask
										)

										if len(__snake_case			) == 0:
														if return_attention_mask:
																		__magic_name__						:						List[str]										=[]
														return processed_features

										# If we have PyTorch/TF tensors or lists as inputs, we cast them as Numpy arrays
										# and rebuild them afterwards if no return_tensors is specified
										# Note that we lose the specific device the tensor may be on for PyTorch

										__magic_name__						:						Optional[int]										=required_input[0]
										if isinstance(__snake_case     ,			(list, tuple)			):
														# first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
														__magic_name__						:						Optional[Any]										=0
														while len(required_input[index]			) == 0:
																		index += 1
														if index < len(__snake_case			):
																		__magic_name__						:						List[str]										=required_input[index][0]

										if return_tensors is None:
														if is_tf_tensor(__snake_case			):
																		__magic_name__						:						int										="""tf"""
														elif is_torch_tensor(__snake_case			):
																		__magic_name__						:						str										="""pt"""
														elif isinstance(__snake_case     ,			(int, float, list, tuple, np.ndarray)			):
																		__magic_name__						:						List[Any]										="""np"""
														else:
																		raise ValueError(
																		    f"type of {first_element} unknown: {type(__snake_case			)}. "
																		    """Should be one of a python, numpy, pytorch or tensorflow object."""			)

										for key, value in processed_features.items():
														if isinstance(value[0]     ,			(int, float)			):
																		__magic_name__						:						List[str]										=to_numpy(__snake_case			)
														else:
																		__magic_name__						:						str										=[to_numpy(__snake_case			) for v in value]

        # Convert padding_strategy in PaddingStrategy
										__magic_name__						:						Dict										=self._get_padding_strategies(padding=__snake_case     ,			max_length=__snake_case			)

										__magic_name__						:						Optional[Any]										=processed_features[self.model_input_names[0]]

										__magic_name__						:						Dict										=len(__snake_case			)
										if not all(len(__snake_case			) == batch_size for v in processed_features.values()			):
														raise ValueError("""Some items in the output dictionary have a different batch size than others."""			)

										__magic_name__						:						Optional[int]										=[]
										for i in range(__snake_case			):
														__magic_name__						:						Any										={k: v[i] for k, v in processed_features.items()}
														# truncation
														__magic_name__						:						List[str]										=self._truncate(
														    __snake_case     ,			max_length=__snake_case     ,			pad_to_multiple_of=__snake_case     ,			truncation=__snake_case     ,			)
														truncated_inputs.append(__snake_case			)

										if padding_strategy == PaddingStrategy.LONGEST:
														# make sure that `max_length` cannot be longer than the longest truncated length
														__magic_name__						:						Optional[int]										=max(len(input_slice[self.model_input_names[0]]			) for input_slice in truncated_inputs			)
														__magic_name__						:						Tuple										=PaddingStrategy.MAX_LENGTH

										__magic_name__						:						str										={}
										for i in range(__snake_case			):
														# padding
														__magic_name__						:						List[str]										=self._pad(
														    truncated_inputs[i]     ,			max_length=__snake_case     ,			padding_strategy=__snake_case     ,			pad_to_multiple_of=__snake_case     ,			return_attention_mask=__snake_case     ,			)

														for key, value in outputs.items():
																		if key not in batch_outputs:
																						__magic_name__						:						Dict										=[]
																		if value.dtype is np.dtype(np.floataa			):
																						__magic_name__						:						Optional[int]										=value.astype(np.floataa			)
																		batch_outputs[key].append(__snake_case			)

										return BatchFeature(__snake_case     ,			tensor_type=__snake_case			)

						def      A__    (		self :Any     ,			__snake_case :Union[Dict[str, np.ndarray], BatchFeature]     ,			__snake_case :Optional[int] = None     ,			__snake_case :PaddingStrategy = PaddingStrategy.DO_NOT_PAD     ,			__snake_case :Optional[int] = None     ,			__snake_case :Optional[bool] = None     ,			):
										'''simple docstring'''

										__magic_name__						:						Dict										=processed_features[self.model_input_names[0]]

										if padding_strategy == PaddingStrategy.LONGEST:
														__magic_name__						:						Any										=len(__snake_case			)

										if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
														__magic_name__						:						Dict										=((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

										__magic_name__						:						List[Any]										=padding_strategy != PaddingStrategy.DO_NOT_PAD and len(__snake_case			) < max_length

										if return_attention_mask and "attention_mask" not in processed_features:
														__magic_name__						:						int										=np.ones(len(__snake_case			)     ,			dtype=np.intaa			)

										if needs_to_be_padded:
														__magic_name__						:						List[Any]										=max_length - len(__snake_case			)
														if self.padding_side == "right":
																		if return_attention_mask:
																						__magic_name__						:						str										=np.pad(
																						    processed_features["""attention_mask"""]     ,			(0, difference)			)
																		__magic_name__						:						Tuple										=((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference)
																		__magic_name__						:						str										=np.pad(
																		    __snake_case     ,			__snake_case     ,			"""constant"""     ,			constant_values=self.padding_value			)
														elif self.padding_side == "left":
																		if return_attention_mask:
																						__magic_name__						:						str										=np.pad(
																						    processed_features["""attention_mask"""]     ,			(difference, 0)			)
																		__magic_name__						:						Optional[int]										=((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0)
																		__magic_name__						:						List[Any]										=np.pad(
																		    __snake_case     ,			__snake_case     ,			"""constant"""     ,			constant_values=self.padding_value			)
														else:
																		raise ValueError("""Invalid padding strategy:""" + str(self.padding_side			)			)

										return processed_features

						def      A__    (		self :Optional[Any]     ,			__snake_case :Union[Dict[str, np.ndarray], BatchFeature]     ,			__snake_case :Optional[int] = None     ,			__snake_case :Optional[int] = None     ,			__snake_case :Optional[bool] = None     ,			):
										'''simple docstring'''

										if not truncation:
														return processed_features
										elif truncation and max_length is None:
														raise ValueError("""When setting ``truncation=True``, make sure that ``max_length`` is defined."""			)

										__magic_name__						:						Union[str, Any]										=processed_features[self.model_input_names[0]]

										# find `max_length` that fits `pad_to_multiple_of`
										if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
														__magic_name__						:						List[str]										=((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

										__magic_name__						:						Any										=len(__snake_case			) > max_length

										if needs_to_be_truncated:
														__magic_name__						:						List[Any]										=processed_features[self.model_input_names[0]][:max_length]
														if "attention_mask" in processed_features:
																		__magic_name__						:						List[str]										=processed_features["""attention_mask"""][:max_length]

										return processed_features


						def      A__    (		self :List[Any]     ,			__snake_case :str=False     ,			__snake_case :Optional[int]=None			):
										'''simple docstring'''

										if padding is not False:
														if padding is True:
																		__magic_name__						:						Union[str, Any]										=PaddingStrategy.LONGEST  # Default to pad to the longest sequence in the batch
														elif not isinstance(__snake_case     ,			__snake_case			):
																		__magic_name__						:						Optional[int]										=PaddingStrategy(__snake_case			)
														elif isinstance(__snake_case     ,			__snake_case			):
																		__magic_name__						:						Any										=padding
										else:
														__magic_name__						:						Any										=PaddingStrategy.DO_NOT_PAD

										# Set max length if needed
										if max_length is None:
														if padding_strategy == PaddingStrategy.MAX_LENGTH:
																		raise ValueError(
																		    f"When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined"			)

        # Test if we have a padding value
										if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):
														raise ValueError(
														    """Asking to pad but the feature_extractor does not have a padding value. Please select a value to use"""
														    """ as `padding_value`. For example: `feature_extractor.padding_value = 0.0`."""			)

										return padding_strategy




