





import inspect
import unittest

import numpy as np

from tests.test_modeling_common import floats_tensor
from transformers import MaskaFormerConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import MaskaFormerForUniversalSegmentation, MaskaFormerModel

						if is_vision_available():
												from transformers import MaskaFormerImageProcessor

if is_vision_available():
						from PIL import Image

class 	UpperCamelCase__				:



					'''simple docstring'''

					def __init__(							self		:					Tuple		,lowerCamelCase__		:					Tuple		,lowerCamelCase__		:					Tuple=2		,lowerCamelCase__		:					Optional[int]=True		,lowerCamelCase__		:					int=False		,lowerCamelCase__		:					List[str]=10		,lowerCamelCase__		:					int=3		,lowerCamelCase__		:					Optional[Any]=32 * 8		,lowerCamelCase__		:					Tuple=32 * 8		,lowerCamelCase__		:					Tuple=4		,lowerCamelCase__		:					int=64		,)  ->	List[Any]:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE	=		parent
											SCREAMING_SNAKE_CASE	=		batch_size
											SCREAMING_SNAKE_CASE	=		is_training
											SCREAMING_SNAKE_CASE	=		use_auxiliary_loss
											SCREAMING_SNAKE_CASE	=		num_queries
											SCREAMING_SNAKE_CASE	=		num_channels
											SCREAMING_SNAKE_CASE	=		min_size
											SCREAMING_SNAKE_CASE	=		max_size
											SCREAMING_SNAKE_CASE	=		num_labels
											SCREAMING_SNAKE_CASE	=		hidden_dim
											SCREAMING_SNAKE_CASE	=		hidden_dim






					def        SCREAMING_SNAKE_CASE__						(							self		:					Union[str, Any] )  ->	str:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE	=		floats_tensor([self.batch_size, self.num_channels, self.min_size, self.max_size] ).to(
											    lowerCamelCase__ )

											SCREAMING_SNAKE_CASE	=		torch.ones([self.batch_size, self.min_size, self.max_size]		,device=lowerCamelCase__ )

											SCREAMING_SNAKE_CASE	=		(
											    torch.rand([self.batch_size, self.num_labels, self.min_size, self.max_size]		,device=lowerCamelCase__ ) > 0.5
											).float()
											SCREAMING_SNAKE_CASE	=		(torch.rand((self.batch_size, self.num_labels)		,device=lowerCamelCase__ ) > 0.5).long()

											SCREAMING_SNAKE_CASE	=		self.get_config()
											return config, pixel_values, pixel_mask, mask_labels, class_labels






					def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[int] )  ->	int:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE	=		MaskaFormerConfig(
											    hidden_size=self.hidden_dim		,)
											SCREAMING_SNAKE_CASE	=		self.num_queries
											SCREAMING_SNAKE_CASE	=		self.num_labels

											SCREAMING_SNAKE_CASE	=		[1, 1, 1, 1]
											SCREAMING_SNAKE_CASE	=		self.num_channels

											SCREAMING_SNAKE_CASE	=		64
											SCREAMING_SNAKE_CASE	=		128
											SCREAMING_SNAKE_CASE	=		self.hidden_dim
											SCREAMING_SNAKE_CASE	=		self.hidden_dim
											SCREAMING_SNAKE_CASE	=		self.hidden_dim
											return config






					def        SCREAMING_SNAKE_CASE__						(							self		:					str )  ->	Dict:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE	=		self.prepare_config_and_inputs()
											SCREAMING_SNAKE_CASE	=		{"""pixel_values""": pixel_values, """pixel_mask""": pixel_mask}
											return config, inputs_dict






					def        SCREAMING_SNAKE_CASE__						(							self		:					List[str]		,lowerCamelCase__		:					List[str]		,lowerCamelCase__		:					Dict )  ->	str:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE	=		output.encoder_hidden_states
											SCREAMING_SNAKE_CASE	=		output.pixel_decoder_hidden_states
											SCREAMING_SNAKE_CASE	=		output.transformer_decoder_hidden_states

											self.parent.assertTrue(len(lowerCamelCase__ )		,len(config.backbone_config.depths ) )
											self.parent.assertTrue(len(lowerCamelCase__ )		,len(config.backbone_config.depths ) )
											self.parent.assertTrue(len(lowerCamelCase__ )		,config.decoder_layers )






					def        SCREAMING_SNAKE_CASE__						(							self		:					Union[str, Any]		,lowerCamelCase__		:					Optional[int]		,lowerCamelCase__		:					Any		,lowerCamelCase__		:					int		,lowerCamelCase__		:					Optional[int]=False )  ->	int:




											'''simple docstring'''


											with torch.no_grad():
																	SCREAMING_SNAKE_CASE	=		MaskaFormerModel(config=lowerCamelCase__ )
																	model.to(lowerCamelCase__ )
																	model.eval()

																	SCREAMING_SNAKE_CASE	=		model(pixel_values=lowerCamelCase__		,pixel_mask=lowerCamelCase__ )
																	SCREAMING_SNAKE_CASE	=		model(lowerCamelCase__		,output_hidden_states=lowerCamelCase__ )

											self.parent.assertEqual(
											    output.transformer_decoder_last_hidden_state.shape		,(self.batch_size, self.num_queries, self.hidden_dim)		,)
											# let's ensure the other two hidden state exists
											self.parent.assertTrue(output.pixel_decoder_last_hidden_state is not None )
											self.parent.assertTrue(output.encoder_last_hidden_state is not None )

											if output_hidden_states:
																	self.check_output_hidden_state(lowerCamelCase__		,lowerCamelCase__ )






					def        SCREAMING_SNAKE_CASE__						(							self		:					Any		,lowerCamelCase__		:					Any		,lowerCamelCase__		:					List[Any]		,lowerCamelCase__		:					Union[str, Any]		,lowerCamelCase__		:					Tuple		,lowerCamelCase__		:					Any )  ->	Optional[int]:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE	=		MaskaFormerForUniversalSegmentation(config=lowerCamelCase__ )
											model.to(lowerCamelCase__ )
											model.eval()

											def comm_check_on_output(lowerCamelCase__		:					Optional[int] ):
																	# let's still check that all the required stuff is there
																	self.parent.assertTrue(result.transformer_decoder_last_hidden_state is not None )
																	self.parent.assertTrue(result.pixel_decoder_last_hidden_state is not None )
																	self.parent.assertTrue(result.encoder_last_hidden_state is not None )
																	# okay, now we need to check the logits shape
																	# due to the encoder compression, masks have a //4 spatial size
																	self.parent.assertEqual(
																	    result.masks_queries_logits.shape		,(self.batch_size, self.num_queries, self.min_size // 4, self.max_size // 4)		,)
																	# + 1 for null class
																	self.parent.assertEqual(
																	    result.class_queries_logits.shape		,(self.batch_size, self.num_queries, self.num_labels + 1) )

											with torch.no_grad():
																	SCREAMING_SNAKE_CASE	=		model(pixel_values=lowerCamelCase__		,pixel_mask=lowerCamelCase__ )
																	SCREAMING_SNAKE_CASE	=		model(lowerCamelCase__ )

																	comm_check_on_output(lowerCamelCase__ )

																	SCREAMING_SNAKE_CASE	=		model(
																	    pixel_values=lowerCamelCase__		,pixel_mask=lowerCamelCase__		,mask_labels=lowerCamelCase__		,class_labels=lowerCamelCase__ )

											comm_check_on_output(lowerCamelCase__ )

											self.parent.assertTrue(result.loss is not None )
											self.parent.assertEqual(result.loss.shape		,torch.Size([1] ) )

@require_torch
class 	UpperCamelCase__				(   lowerCAmelCase_							,   lowerCAmelCase_							,   unittest.TestCase     ):



					'''simple docstring'''

					__snake_case					:			Union[str, Any]				=    (MaskaFormerModel, MaskaFormerForUniversalSegmentation) if is_torch_available() else ()
					__snake_case					:			Optional[Any]				=    {"feature-extraction": MaskaFormerModel} if is_torch_available() else {}

					__snake_case					:			Dict				=    False
					__snake_case					:			Tuple				=    False
					__snake_case					:			Union[str, Any]				=    False
					__snake_case					:			Optional[Any]				=    False






					def        SCREAMING_SNAKE_CASE__						(							self		:					int )  ->	Optional[int]:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE	=		MaskaFormerModelTester(self )
											SCREAMING_SNAKE_CASE	=		ConfigTester(self		,config_class=lowerCamelCase__		,has_text_modality=lowerCamelCase__ )






					def        SCREAMING_SNAKE_CASE__						(							self		:					int )  ->	Dict:




											'''simple docstring'''


											self.config_tester.run_common_tests()






					def        SCREAMING_SNAKE_CASE__						(							self		:					str )  ->	Union[str, Any]:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE	=		self.model_tester.prepare_config_and_inputs_for_common()
											self.model_tester.create_and_check_maskaformer_model(lowerCamelCase__		,**lowerCamelCase__		,output_hidden_states=lowerCamelCase__ )






					def        SCREAMING_SNAKE_CASE__						(							self		:					List[str] )  ->	Optional[Any]:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE	=		self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_maskaformer_instance_segmentation_head_model(*lowerCamelCase__ )






					@unittest.skip(reason="""Mask2Former does not use inputs_embeds""" )
					def        SCREAMING_SNAKE_CASE__						(							self		:					Dict )  ->	int:




											'''simple docstring'''


											pass






					@unittest.skip(reason="""Mask2Former does not have a get_input_embeddings method""" )
					def        SCREAMING_SNAKE_CASE__						(							self		:					Tuple )  ->	Tuple:




											'''simple docstring'''


											pass






					@unittest.skip(reason="""Mask2Former is not a generative model""" )
					def        SCREAMING_SNAKE_CASE__						(							self		:					Tuple )  ->	Dict:




											'''simple docstring'''


											pass






					@unittest.skip(reason="""Mask2Former does not use token embeddings""" )
					def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[Any] )  ->	List[Any]:




											'''simple docstring'''


											pass






					@require_torch_multi_gpu
					@unittest.skip(
					    reason="""Mask2Former has some layers using `add_module` which doesn't work well with `nn.DataParallel`""" )
					def        SCREAMING_SNAKE_CASE__						(							self		:					Tuple )  ->	Any:




											'''simple docstring'''


											pass






					@unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests.""" )
					def        SCREAMING_SNAKE_CASE__						(							self		:					List[str] )  ->	Tuple:




											'''simple docstring'''


											pass






					def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[int] )  ->	List[Any]:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE	=		self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																	SCREAMING_SNAKE_CASE	=		model_class(lowerCamelCase__ )
																	SCREAMING_SNAKE_CASE	=		inspect.signature(model.forward )
																	# signature.parameters is an OrderedDict => so arg_names order is deterministic
																	SCREAMING_SNAKE_CASE	=		[*signature.parameters.keys()]

																	SCREAMING_SNAKE_CASE	=		["""pixel_values"""]
																	self.assertListEqual(arg_names[:1]		,lowerCamelCase__ )






					@slow
					def        SCREAMING_SNAKE_CASE__						(							self		:					Any )  ->	Any:




											'''simple docstring'''


											for model_name in ["facebook/mask2former-swin-small-coco-instance"]:
																	SCREAMING_SNAKE_CASE	=		MaskaFormerModel.from_pretrained(lowerCamelCase__ )
																	self.assertIsNotNone(lowerCamelCase__ )






					def        SCREAMING_SNAKE_CASE__						(							self		:					Any )  ->	Union[str, Any]:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE	=		(self.model_tester.min_size,) * 2
											SCREAMING_SNAKE_CASE	=		{
											    """pixel_values""": torch.randn((2, 3, *size)		,device=lowerCamelCase__ ),
											    """mask_labels""": torch.randn((2, 10, *size)		,device=lowerCamelCase__ ),
											    """class_labels""": torch.zeros(2		,10		,device=lowerCamelCase__ ).long(),
											}
											SCREAMING_SNAKE_CASE	=		self.model_tester.get_config()

											SCREAMING_SNAKE_CASE	=		MaskaFormerForUniversalSegmentation(lowerCamelCase__ ).to(lowerCamelCase__ )
											SCREAMING_SNAKE_CASE	=		model(**lowerCamelCase__ )
											self.assertTrue(outputs.loss is not None )






					def        SCREAMING_SNAKE_CASE__						(							self		:					int )  ->	List[str]:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE	=		self.model_tester.prepare_config_and_inputs_for_common()
											self.model_tester.create_and_check_maskaformer_model(lowerCamelCase__		,**lowerCamelCase__		,output_hidden_states=lowerCamelCase__ )






					def        SCREAMING_SNAKE_CASE__						(							self		:					Dict )  ->	Optional[Any]:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE	=		self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																	SCREAMING_SNAKE_CASE	=		model_class(lowerCamelCase__ ).to(lowerCamelCase__ )
																	SCREAMING_SNAKE_CASE	=		model(**lowerCamelCase__		,output_attentions=lowerCamelCase__ )
																	self.assertTrue(outputs.attentions is not None )






					def        SCREAMING_SNAKE_CASE__						(							self		:					Tuple )  ->	List[str]:




											'''simple docstring'''


											if not self.model_tester.is_training:
																	return

											SCREAMING_SNAKE_CASE	=		self.all_model_classes[1]
											SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE	=		self.model_tester.prepare_config_and_inputs()

											SCREAMING_SNAKE_CASE	=		model_class(lowerCamelCase__ )
											model.to(lowerCamelCase__ )
											model.train()

											SCREAMING_SNAKE_CASE	=		model(lowerCamelCase__		,mask_labels=lowerCamelCase__		,class_labels=lowerCamelCase__ ).loss
											loss.backward()






					def        SCREAMING_SNAKE_CASE__						(							self		:					int )  ->	Optional[int]:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE	=		self.all_model_classes[1]
											SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE	=		self.model_tester.prepare_config_and_inputs()
											SCREAMING_SNAKE_CASE	=		True
											SCREAMING_SNAKE_CASE	=		True

											SCREAMING_SNAKE_CASE	=		model_class(lowerCamelCase__ ).to(lowerCamelCase__ )
											model.train()

											SCREAMING_SNAKE_CASE	=		model(lowerCamelCase__		,mask_labels=lowerCamelCase__		,class_labels=lowerCamelCase__ )

											SCREAMING_SNAKE_CASE	=		outputs.encoder_hidden_states[0]
											encoder_hidden_states.retain_grad()

											SCREAMING_SNAKE_CASE	=		outputs.pixel_decoder_hidden_states[0]
											pixel_decoder_hidden_states.retain_grad()

											SCREAMING_SNAKE_CASE	=		outputs.transformer_decoder_hidden_states[0]
											transformer_decoder_hidden_states.retain_grad()

											SCREAMING_SNAKE_CASE	=		outputs.attentions[0]
											attentions.retain_grad()

											outputs.loss.backward(retain_graph=lowerCamelCase__ )

											self.assertIsNotNone(encoder_hidden_states.grad )
											self.assertIsNotNone(pixel_decoder_hidden_states.grad )
											self.assertIsNotNone(transformer_decoder_hidden_states.grad )
											self.assertIsNotNone(attentions.grad )
SCREAMING_SNAKE_CASE_		     = 1e-4



def      __lowercase     (   )				->     Any:


						'''simple docstring'''


						SCREAMING_SNAKE_CASE	=		Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""			)
						return image



@require_vision
@slow
class 	UpperCamelCase__				(   unittest.TestCase     ):



					'''simple docstring'''

					@cached_property
					def        SCREAMING_SNAKE_CASE__						(							self		:					str )  ->	Optional[int]:




											'''simple docstring'''


											return "facebook/mask2former-swin-small-coco-instance"






					@cached_property
					def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[Any] )  ->	Optional[int]:




											'''simple docstring'''


											return MaskaFormerImageProcessor.from_pretrained(self.model_checkpoints ) if is_vision_available() else None






					def        SCREAMING_SNAKE_CASE__						(							self		:					Dict )  ->	int:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE	=		MaskaFormerModel.from_pretrained(self.model_checkpoints ).to(lowerCamelCase__ )
											SCREAMING_SNAKE_CASE	=		self.default_image_processor
											SCREAMING_SNAKE_CASE	=		prepare_img()
											SCREAMING_SNAKE_CASE	=		image_processor(lowerCamelCase__		,return_tensors="""pt""" ).to(lowerCamelCase__ )
											SCREAMING_SNAKE_CASE	=		inputs["""pixel_values"""].shape
											# check size is divisible by 32
											self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0 )
											# check size
											self.assertEqual(lowerCamelCase__		,(1, 3, 384, 384) )

											with torch.no_grad():
																	SCREAMING_SNAKE_CASE	=		model(**lowerCamelCase__ )

											SCREAMING_SNAKE_CASE	=		torch.tensor(
											    [[-0.2790, -1.0717, -1.1668], [-0.5128, -0.3128, -0.4987], [-0.5832, 0.1971, -0.0197]] ).to(lowerCamelCase__ )
											self.assertTrue(
											    torch.allclose(
											        outputs.encoder_last_hidden_state[0, 0, :3, :3]		,lowerCamelCase__		,atol=lowerCamelCase__ ) )

											SCREAMING_SNAKE_CASE	=		torch.tensor(
											    [[0.8973, 1.1847, 1.1776], [1.1934, 1.5040, 1.5128], [1.1153, 1.4486, 1.4951]] ).to(lowerCamelCase__ )
											self.assertTrue(
											    torch.allclose(
											        outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3]		,lowerCamelCase__		,atol=lowerCamelCase__ ) )

											SCREAMING_SNAKE_CASE	=		torch.tensor(
											    [[2.1152, 1.7000, -0.8603], [1.5808, 1.8004, -0.9353], [1.6043, 1.7495, -0.5999]] ).to(lowerCamelCase__ )
											self.assertTrue(
											    torch.allclose(
											        outputs.transformer_decoder_last_hidden_state[0, :3, :3]		,lowerCamelCase__		,atol=lowerCamelCase__ ) )






					def        SCREAMING_SNAKE_CASE__						(							self		:					int )  ->	int:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE	=		MaskaFormerForUniversalSegmentation.from_pretrained(self.model_checkpoints ).to(lowerCamelCase__ ).eval()
											SCREAMING_SNAKE_CASE	=		self.default_image_processor
											SCREAMING_SNAKE_CASE	=		prepare_img()
											SCREAMING_SNAKE_CASE	=		image_processor(lowerCamelCase__		,return_tensors="""pt""" ).to(lowerCamelCase__ )
											SCREAMING_SNAKE_CASE	=		inputs["""pixel_values"""].shape
											# check size is divisible by 32
											self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0 )
											# check size
											self.assertEqual(lowerCamelCase__		,(1, 3, 384, 384) )

											with torch.no_grad():
																	SCREAMING_SNAKE_CASE	=		model(**lowerCamelCase__ )
											# masks_queries_logits
											SCREAMING_SNAKE_CASE	=		outputs.masks_queries_logits
											self.assertEqual(
											    masks_queries_logits.shape		,(1, model.config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4) )
											SCREAMING_SNAKE_CASE	=		[
											    [-8.7839, -9.0056, -8.8121],
											    [-7.4104, -7.0313, -6.5401],
											    [-6.6105, -6.3427, -6.4675],
											]
											SCREAMING_SNAKE_CASE	=		torch.tensor(lowerCamelCase__ ).to(lowerCamelCase__ )
											self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]		,lowerCamelCase__		,atol=lowerCamelCase__ ) )
											# class_queries_logits
											SCREAMING_SNAKE_CASE	=		outputs.class_queries_logits
											self.assertEqual(class_queries_logits.shape		,(1, model.config.num_queries, model.config.num_labels + 1) )
											SCREAMING_SNAKE_CASE	=		torch.tensor(
											    [
											        [1.8324, -8.0835, -4.1922],
											        [0.8450, -9.0050, -3.6053],
											        [0.3045, -7.7293, -3.0275],
											    ] ).to(lowerCamelCase__ )
											self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]		,lowerCamelCase__		,atol=lowerCamelCase__ ) )






					def        SCREAMING_SNAKE_CASE__						(							self		:					int )  ->	Optional[int]:




											'''simple docstring'''


											SCREAMING_SNAKE_CASE	=		MaskaFormerForUniversalSegmentation.from_pretrained(self.model_checkpoints ).to(lowerCamelCase__ ).eval()
											SCREAMING_SNAKE_CASE	=		self.default_image_processor

											SCREAMING_SNAKE_CASE	=		image_processor(
											    [np.zeros((3, 800, 1333) ), np.zeros((3, 800, 1333) )]		,segmentation_maps=[np.zeros((384, 384) ).astype(np.floataa ), np.zeros((384, 384) ).astype(np.floataa )]		,return_tensors="""pt"""		,)

											SCREAMING_SNAKE_CASE	=		inputs["""pixel_values"""].to(lowerCamelCase__ )
											SCREAMING_SNAKE_CASE	=		[el.to(lowerCamelCase__ ) for el in inputs["""mask_labels"""]]
											SCREAMING_SNAKE_CASE	=		[el.to(lowerCamelCase__ ) for el in inputs["""class_labels"""]]

											with torch.no_grad():
																	SCREAMING_SNAKE_CASE	=		model(**lowerCamelCase__ )

											self.assertTrue(outputs.loss is not None )









import inspect
import re
from hashlib import shaaaa
from typing import Dict, List

from .arrow import arrow
from .audiofolder import audiofolder
from .csv import csv
from .imagefolder import imagefolder
from .json import json
from .pandas import pandas
from .parquet import parquet
from .sql import sql  # noqa F401
from .text import text



def      __lowercase     (   _SCREAMING_SNAKE_CASE			)				->     str:


      '''simple docstring'''


      SCREAMING_SNAKE_CASE	=		[]
      for line in lines:
            SCREAMING_SNAKE_CASE	=		re.sub(r"""#.*"""				,		""""""				,		_SCREAMING_SNAKE_CASE			)  # remove comments
            if line:
                  filtered_lines.append(_SCREAMING_SNAKE_CASE			)
      SCREAMING_SNAKE_CASE	=		"""\n""".join(_SCREAMING_SNAKE_CASE			)

      # Make a hash from all this code
      SCREAMING_SNAKE_CASE	=		full_str.encode("""utf-8"""			)
      return shaaaa(_SCREAMING_SNAKE_CASE			).hexdigest()


# get importable module names and hash for caching
SCREAMING_SNAKE_CASE_		     = {
    """csv""": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
    """json""": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
    """pandas""": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
    """parquet""": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
    """arrow""": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
    """text""": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
    """imagefolder""": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
    """audiofolder""": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
}

# Used to infer the module to use based on the data files extensions
SCREAMING_SNAKE_CASE_		     = {
    """.csv""": ("""csv""", {}),
    """.tsv""": ("""csv""", {"""sep""": """\t"""}),
    """.json""": ("""json""", {}),
    """.jsonl""": ("""json""", {}),
    """.parquet""": ("""parquet""", {}),
    """.arrow""": ("""arrow""", {}),
    """.txt""": ("""text""", {}),
}
_EXTENSION_TO_MODULE.update({ext: ("""imagefolder""", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ("""imagefolder""", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext: ("""audiofolder""", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ("""audiofolder""", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
SCREAMING_SNAKE_CASE_		     = {"""imagefolder""", """audiofolder"""}

# Used to filter data files based on extensions given a module name
SCREAMING_SNAKE_CASE_		     = {}
for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
      _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)

_MODULE_TO_EXTENSIONS["imagefolder"].append(""".zip""")



_MODULE_TO_EXTENSIONS["audiofolder"].append(""".zip""")


