import inspect
import unittest

import numpy as np

from tests.test_modeling_common import floats_tensor
from transformers import MaskaFormerConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import MaskaFormerForUniversalSegmentation, MaskaFormerModel

  if is_vision_available():
    from transformers import MaskaFormerImageProcessor

if is_vision_available():
  from PIL import Image
class  A_			:

  def __init__(		self			:   Union[str, Any]  ,				__SCREAMING_SNAKE_CASE			:   Dict  ,				__SCREAMING_SNAKE_CASE			:   Optional[Any]=2  ,				__SCREAMING_SNAKE_CASE			:   Optional[int]=True  ,				__SCREAMING_SNAKE_CASE			:   Optional[int]=False  ,				__SCREAMING_SNAKE_CASE			:   Optional[Any]=10  ,				__SCREAMING_SNAKE_CASE			:   Optional[int]=3  ,				__SCREAMING_SNAKE_CASE			:   Tuple=32 * 8  ,				__SCREAMING_SNAKE_CASE			:   Union[str, Any]=32 * 8  ,				__SCREAMING_SNAKE_CASE			:   List[str]=4  ,				__SCREAMING_SNAKE_CASE			:   List[str]=64  ,				):
   __a							=	parent
   __a							=	batch_size
   __a							=	is_training
   __a							=	use_auxiliary_loss
   __a							=	num_queries
   __a							=	num_channels
   __a							=	min_size
   __a							=	max_size
   __a							=	num_labels
   __a							=	hidden_dim
   __a							=	hidden_dim

  def 		_UpperCAmelCase (		self			:   Dict							):
   __a							=	floats_tensor([self.batch_size, self.num_channels, self.min_size, self.max_size]							).to(
       __SCREAMING_SNAKE_CASE							)

   __a							=	torch.ones([self.batch_size, self.min_size, self.max_size]  ,				device=__SCREAMING_SNAKE_CASE							)

   __a							=	(
       torch.rand([self.batch_size, self.num_labels, self.min_size, self.max_size]  ,				device=__SCREAMING_SNAKE_CASE							) > 0.5
   ).float()
   __a							=	(torch.rand((self.batch_size, self.num_labels)  ,				device=__SCREAMING_SNAKE_CASE							) > 0.5).long()

   __a							=	self.get_config()
   return config, pixel_values, pixel_mask, mask_labels, class_labels

  def 		_UpperCAmelCase (		self			:   Any							):
   __a							=	MaskaFormerConfig(
       hidden_size=self.hidden_dim  ,				)
   __a							=	self.num_queries
   __a							=	self.num_labels

   __a							=	[1, 1, 1, 1]
   __a							=	self.num_channels

   __a							=	64
   __a							=	1_28
   __a							=	self.hidden_dim
   __a							=	self.hidden_dim
   __a							=	self.hidden_dim
   return config

  def 		_UpperCAmelCase (		self			:   str							):
   __a							,  __a							,  __a							,  __a							,  __a							=	self.prepare_config_and_inputs()
   __a							=	{"pixel_values": pixel_values, "pixel_mask": pixel_mask}
   return config, inputs_dict

  def 		_UpperCAmelCase (		self			:   int  ,				__SCREAMING_SNAKE_CASE			:   Optional[Any]  ,				__SCREAMING_SNAKE_CASE			:   Optional[Any]							):
   __a							=	output.encoder_hidden_states
   __a							=	output.pixel_decoder_hidden_states
   __a							=	output.transformer_decoder_hidden_states

   self.parent.assertTrue(len(__SCREAMING_SNAKE_CASE							)  ,				len(config.backbone_config.depths							)							)
   self.parent.assertTrue(len(__SCREAMING_SNAKE_CASE							)  ,				len(config.backbone_config.depths							)							)
   self.parent.assertTrue(len(__SCREAMING_SNAKE_CASE							)  ,				config.decoder_layers							)

  def 		_UpperCAmelCase (		self			:   Dict  ,				__SCREAMING_SNAKE_CASE			:   int  ,				__SCREAMING_SNAKE_CASE			:   Dict  ,				__SCREAMING_SNAKE_CASE			:   Optional[int]  ,				__SCREAMING_SNAKE_CASE			:   str=False							):
   with torch.no_grad():
    __a							=	MaskaFormerModel(config=__SCREAMING_SNAKE_CASE							)
    model.to(__SCREAMING_SNAKE_CASE							)
    model.eval()

    __a							=	model(pixel_values=__SCREAMING_SNAKE_CASE  ,				pixel_mask=__SCREAMING_SNAKE_CASE							)
    __a							=	model(__SCREAMING_SNAKE_CASE  ,				output_hidden_states=__SCREAMING_SNAKE_CASE							)

   self.parent.assertEqual(
       output.transformer_decoder_last_hidden_state.shape  ,				(self.batch_size, self.num_queries, self.hidden_dim)  ,				)
   # let's ensure the other two hidden state exists
   self.parent.assertTrue(output.pixel_decoder_last_hidden_state is not None							)
   self.parent.assertTrue(output.encoder_last_hidden_state is not None							)

   if output_hidden_states:
    self.check_output_hidden_state(__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE							)

  def 		_UpperCAmelCase (		self			:   Dict  ,				__SCREAMING_SNAKE_CASE			:   str  ,				__SCREAMING_SNAKE_CASE			:   Any  ,				__SCREAMING_SNAKE_CASE			:   Tuple  ,				__SCREAMING_SNAKE_CASE			:   Union[str, Any]  ,				__SCREAMING_SNAKE_CASE			:   List[Any]							):
   __a							=	MaskaFormerForUniversalSegmentation(config=__SCREAMING_SNAKE_CASE							)
   model.to(__SCREAMING_SNAKE_CASE							)
   model.eval()

   def comm_check_on_output(__SCREAMING_SNAKE_CASE			:   str							):
    # let's still check that all the required stuff is there
    self.parent.assertTrue(result.transformer_decoder_last_hidden_state is not None							)
    self.parent.assertTrue(result.pixel_decoder_last_hidden_state is not None							)
    self.parent.assertTrue(result.encoder_last_hidden_state is not None							)
    # okay, now we need to check the logits shape
    # due to the encoder compression, masks have a //4 spatial size
    self.parent.assertEqual(
        result.masks_queries_logits.shape  ,				(self.batch_size, self.num_queries, self.min_size // 4, self.max_size // 4)  ,				)
    # + 1 for null class
    self.parent.assertEqual(
        result.class_queries_logits.shape  ,				(self.batch_size, self.num_queries, self.num_labels + 1)							)

   with torch.no_grad():
    __a							=	model(pixel_values=__SCREAMING_SNAKE_CASE  ,				pixel_mask=__SCREAMING_SNAKE_CASE							)
    __a							=	model(__SCREAMING_SNAKE_CASE							)

    comm_check_on_output(__SCREAMING_SNAKE_CASE							)

    __a							=	model(
        pixel_values=__SCREAMING_SNAKE_CASE  ,				pixel_mask=__SCREAMING_SNAKE_CASE  ,				mask_labels=__SCREAMING_SNAKE_CASE  ,				class_labels=__SCREAMING_SNAKE_CASE							)

   comm_check_on_output(__SCREAMING_SNAKE_CASE							)

   self.parent.assertTrue(result.loss is not None							)
   self.parent.assertEqual(result.loss.shape  ,				torch.Size([1]							)							)
@require_torch
class  A_			(							a_      ,							a_      ,							unittest.TestCase			):
  _SCREAMING_SNAKE_CASE     					=      (MaskaFormerModel, MaskaFormerForUniversalSegmentation) if is_torch_available() else ()
  _SCREAMING_SNAKE_CASE     					=      {"""feature-extraction""": MaskaFormerModel} if is_torch_available() else {}

  _SCREAMING_SNAKE_CASE     					=      False
  _SCREAMING_SNAKE_CASE     					=      False
  _SCREAMING_SNAKE_CASE     					=      False
  _SCREAMING_SNAKE_CASE     					=      False

  def 		_UpperCAmelCase (		self			:   List[Any]							):
   __a							=	MaskaFormerModelTester(self							)
   __a							=	ConfigTester(self  ,				config_class=__SCREAMING_SNAKE_CASE  ,				has_text_modality=__SCREAMING_SNAKE_CASE							)

  def 		_UpperCAmelCase (		self			:   Union[str, Any]							):
   self.config_tester.run_common_tests()

  def 		_UpperCAmelCase (		self			:   Union[str, Any]							):
   __a							,  __a							=	self.model_tester.prepare_config_and_inputs_for_common()
   self.model_tester.create_and_check_maskaformer_model(__SCREAMING_SNAKE_CASE  ,				**__SCREAMING_SNAKE_CASE  ,				output_hidden_states=__SCREAMING_SNAKE_CASE							)

  def 		_UpperCAmelCase (		self			:   Optional[int]							):
   __a							=	self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_maskaformer_instance_segmentation_head_model(*__SCREAMING_SNAKE_CASE							)

  @unittest.skip(reason="Mask2Former does not use inputs_embeds"							)
  def 		_UpperCAmelCase (		self			:   List[Any]							):
   pass

  @unittest.skip(reason="Mask2Former does not have a get_input_embeddings method"							)
  def 		_UpperCAmelCase (		self			:   Union[str, Any]							):
   pass

  @unittest.skip(reason="Mask2Former is not a generative model"							)
  def 		_UpperCAmelCase (		self			:   List[Any]							):
   pass

  @unittest.skip(reason="Mask2Former does not use token embeddings"							)
  def 		_UpperCAmelCase (		self			:   Optional[int]							):
   pass

  @require_torch_multi_gpu
  @unittest.skip(
      reason="Mask2Former has some layers using `add_module` which doesn't work well with `nn.DataParallel`"							)
  def 		_UpperCAmelCase (		self			:   Optional[int]							):
   pass

  @unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."							)
  def 		_UpperCAmelCase (		self			:   Union[str, Any]							):
   pass

  def 		_UpperCAmelCase (		self			:   List[str]							):
   __a							,  __a							=	self.model_tester.prepare_config_and_inputs_for_common()

   for model_class in self.all_model_classes:
    __a							=	model_class(__SCREAMING_SNAKE_CASE							)
    __a							=	inspect.signature(model.forward							)
    # signature.parameters is an OrderedDict => so arg_names order is deterministic
    __a							=	[*signature.parameters.keys()]

    __a							=	["pixel_values"]
    self.assertListEqual(arg_names[:1]  ,				__SCREAMING_SNAKE_CASE							)

  @slow
  def 		_UpperCAmelCase (		self			:   int							):
   for model_name in ["facebook/mask2former-swin-small-coco-instance"]:
    __a							=	MaskaFormerModel.from_pretrained(__SCREAMING_SNAKE_CASE							)
    self.assertIsNotNone(__SCREAMING_SNAKE_CASE							)

  def 		_UpperCAmelCase (		self			:   List[str]							):
   __a							=	(self.model_tester.min_size,) * 2
   __a							=	{
       "pixel_values": torch.randn((2, 3, *size)  ,				device=__SCREAMING_SNAKE_CASE							),
       "mask_labels": torch.randn((2, 10, *size)  ,				device=__SCREAMING_SNAKE_CASE							),
       "class_labels": torch.zeros(2  ,				10  ,				device=__SCREAMING_SNAKE_CASE							).long(),
   }
   __a							=	self.model_tester.get_config()

   __a							=	MaskaFormerForUniversalSegmentation(__SCREAMING_SNAKE_CASE							).to(__SCREAMING_SNAKE_CASE							)
   __a							=	model(**__SCREAMING_SNAKE_CASE							)
   self.assertTrue(outputs.loss is not None							)

  def 		_UpperCAmelCase (		self			:   Optional[Any]							):
   __a							,  __a							=	self.model_tester.prepare_config_and_inputs_for_common()
   self.model_tester.create_and_check_maskaformer_model(__SCREAMING_SNAKE_CASE  ,				**__SCREAMING_SNAKE_CASE  ,				output_hidden_states=__SCREAMING_SNAKE_CASE							)

  def 		_UpperCAmelCase (		self			:   Optional[int]							):
   __a							,  __a							=	self.model_tester.prepare_config_and_inputs_for_common()

   for model_class in self.all_model_classes:
    __a							=	model_class(__SCREAMING_SNAKE_CASE							).to(__SCREAMING_SNAKE_CASE							)
    __a							=	model(**__SCREAMING_SNAKE_CASE  ,				output_attentions=__SCREAMING_SNAKE_CASE							)
    self.assertTrue(outputs.attentions is not None							)

  def 		_UpperCAmelCase (		self			:   Any							):
   if not self.model_tester.is_training:
    return

   __a							=	self.all_model_classes[1]
   __a							,  __a							,  __a							,  __a							,  __a							=	self.model_tester.prepare_config_and_inputs()

   __a							=	model_class(__SCREAMING_SNAKE_CASE							)
   model.to(__SCREAMING_SNAKE_CASE							)
   model.train()

   __a							=	model(__SCREAMING_SNAKE_CASE  ,				mask_labels=__SCREAMING_SNAKE_CASE  ,				class_labels=__SCREAMING_SNAKE_CASE							).loss
   loss.backward()

  def 		_UpperCAmelCase (		self			:   str							):
   __a							=	self.all_model_classes[1]
   __a							,  __a							,  __a							,  __a							,  __a							=	self.model_tester.prepare_config_and_inputs()
   __a							=	True
   __a							=	True

   __a							=	model_class(__SCREAMING_SNAKE_CASE							).to(__SCREAMING_SNAKE_CASE							)
   model.train()

   __a							=	model(__SCREAMING_SNAKE_CASE  ,				mask_labels=__SCREAMING_SNAKE_CASE  ,				class_labels=__SCREAMING_SNAKE_CASE							)

   __a							=	outputs.encoder_hidden_states[0]
   encoder_hidden_states.retain_grad()

   __a							=	outputs.pixel_decoder_hidden_states[0]
   pixel_decoder_hidden_states.retain_grad()

   __a							=	outputs.transformer_decoder_hidden_states[0]
   transformer_decoder_hidden_states.retain_grad()

   __a							=	outputs.attentions[0]
   attentions.retain_grad()

   outputs.loss.backward(retain_graph=__SCREAMING_SNAKE_CASE							)

   self.assertIsNotNone(encoder_hidden_states.grad							)
   self.assertIsNotNone(pixel_decoder_hidden_states.grad							)
   self.assertIsNotNone(transformer_decoder_hidden_states.grad							)
   self.assertIsNotNone(attentions.grad							)
SCREAMING_SNAKE_CASE			:						Any				  =		1e-4



def 	__A		(      ):



 """simple docstring"""


 __a							=	Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"       )
 return image






@require_vision
@slow
class  A_			(							unittest.TestCase			):

  @cached_property
  def 		_UpperCAmelCase (		self			:   int							):
   return "facebook/mask2former-swin-small-coco-instance"

  @cached_property
  def 		_UpperCAmelCase (		self			:   Union[str, Any]							):
   return MaskaFormerImageProcessor.from_pretrained(self.model_checkpoints							) if is_vision_available() else None

  def 		_UpperCAmelCase (		self			:   Optional[Any]							):
   __a							=	MaskaFormerModel.from_pretrained(self.model_checkpoints							).to(__SCREAMING_SNAKE_CASE							)
   __a							=	self.default_image_processor
   __a							=	prepare_img()
   __a							=	image_processor(__SCREAMING_SNAKE_CASE  ,				return_tensors="pt"							).to(__SCREAMING_SNAKE_CASE							)
   __a							=	inputs["pixel_values"].shape
   # check size is divisible by 32
   self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0							)
   # check size
   self.assertEqual(__SCREAMING_SNAKE_CASE  ,				(1, 3, 3_84, 3_84)							)

   with torch.no_grad():
    __a							=	model(**__SCREAMING_SNAKE_CASE							)

   __a							=	torch.tensor(
       [[-0.27_90, -1.07_17, -1.16_68], [-0.51_28, -0.31_28, -0.49_87], [-0.58_32, 0.19_71, -0.01_97]]							).to(__SCREAMING_SNAKE_CASE							)
   self.assertTrue(
       torch.allclose(
           outputs.encoder_last_hidden_state[0, 0, :3, :3]  ,				__SCREAMING_SNAKE_CASE  ,				atol=__SCREAMING_SNAKE_CASE							)							)

   __a							=	torch.tensor(
       [[0.89_73, 1.18_47, 1.17_76], [1.19_34, 1.50_40, 1.51_28], [1.11_53, 1.44_86, 1.49_51]]							).to(__SCREAMING_SNAKE_CASE							)
   self.assertTrue(
       torch.allclose(
           outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3]  ,				__SCREAMING_SNAKE_CASE  ,				atol=__SCREAMING_SNAKE_CASE							)							)

   __a							=	torch.tensor(
       [[2.11_52, 1.70_00, -0.86_03], [1.58_08, 1.80_04, -0.93_53], [1.60_43, 1.74_95, -0.59_99]]							).to(__SCREAMING_SNAKE_CASE							)
   self.assertTrue(
       torch.allclose(
           outputs.transformer_decoder_last_hidden_state[0, :3, :3]  ,				__SCREAMING_SNAKE_CASE  ,				atol=__SCREAMING_SNAKE_CASE							)							)

  def 		_UpperCAmelCase (		self			:   int							):
   __a							=	MaskaFormerForUniversalSegmentation.from_pretrained(self.model_checkpoints							).to(__SCREAMING_SNAKE_CASE							).eval()
   __a							=	self.default_image_processor
   __a							=	prepare_img()
   __a							=	image_processor(__SCREAMING_SNAKE_CASE  ,				return_tensors="pt"							).to(__SCREAMING_SNAKE_CASE							)
   __a							=	inputs["pixel_values"].shape
   # check size is divisible by 32
   self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0							)
   # check size
   self.assertEqual(__SCREAMING_SNAKE_CASE  ,				(1, 3, 3_84, 3_84)							)

   with torch.no_grad():
    __a							=	model(**__SCREAMING_SNAKE_CASE							)
   # masks_queries_logits
   __a							=	outputs.masks_queries_logits
   self.assertEqual(
       masks_queries_logits.shape  ,				(1, model.config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)							)
   __a							=	[
       [-8.78_39, -9.00_56, -8.81_21],
       [-7.41_04, -7.03_13, -6.54_01],
       [-6.61_05, -6.34_27, -6.46_75],
   ]
   __a							=	torch.tensor(__SCREAMING_SNAKE_CASE							).to(__SCREAMING_SNAKE_CASE							)
   self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]  ,				__SCREAMING_SNAKE_CASE  ,				atol=__SCREAMING_SNAKE_CASE							)							)
   # class_queries_logits
   __a							=	outputs.class_queries_logits
   self.assertEqual(class_queries_logits.shape  ,				(1, model.config.num_queries, model.config.num_labels + 1)							)
   __a							=	torch.tensor(
       [
           [1.83_24, -8.08_35, -4.19_22],
           [0.84_50, -9.00_50, -3.60_53],
           [0.30_45, -7.72_93, -3.02_75],
       ]							).to(__SCREAMING_SNAKE_CASE							)
   self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]  ,				__SCREAMING_SNAKE_CASE  ,				atol=__SCREAMING_SNAKE_CASE							)							)

  def 		_UpperCAmelCase (		self			:   Dict							):
   __a							=	MaskaFormerForUniversalSegmentation.from_pretrained(self.model_checkpoints							).to(__SCREAMING_SNAKE_CASE							).eval()
   __a							=	self.default_image_processor

   __a							=	image_processor(
       [np.zeros((3, 8_00, 13_33)							), np.zeros((3, 8_00, 13_33)							)]  ,				segmentation_maps=[np.zeros((3_84, 3_84)							).astype(np.floataa							), np.zeros((3_84, 3_84)							).astype(np.floataa							)]  ,				return_tensors="pt"  ,				)

   __a							=	inputs["pixel_values"].to(__SCREAMING_SNAKE_CASE							)
   __a							=	[el.to(__SCREAMING_SNAKE_CASE							) for el in inputs["mask_labels"]]
   __a							=	[el.to(__SCREAMING_SNAKE_CASE							) for el in inputs["class_labels"]]

   with torch.no_grad():
    __a							=	model(**__SCREAMING_SNAKE_CASE							)

   self.assertTrue(outputs.loss is not None							)



import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config



@pytest.fixture(scope="session"       )
def 	__A		(      ):



 """simple docstring"""


 __a							=	10
 __a							=	datasets.Features(
     {
         "tokens": datasets.Sequence(datasets.Value("string"       )       ),
         "labels": datasets.Sequence(datasets.ClassLabel(names=["negative", "positive"]       )       ),
         "answers": datasets.Sequence(
             {
                 "text": datasets.Value("string"       ),
                 "answer_start": datasets.Value("int32"       ),
             }       ),
         "id": datasets.Value("int64"       ),
     }       )
 __a							=	datasets.Dataset.from_dict(
     {
         "tokens": [["foo"] * 5] * n,
         "labels": [[1] * 5] * n,
         "answers": [{"answer_start": [97], "text": ["1976"]}] * 10,
         "id": list(range(_A       )       ),
     }				,					features=_A				,					)
 return dataset



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "file.arrow"       )
 dataset.map(cache_file_name=_A       )
 return filename


# FILE_CONTENT + files


SCREAMING_SNAKE_CASE			:						Tuple				  =		"""\
    Text data.
    Second line of data."""



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt"
 __a							=	FILE_CONTENT
 with open(_A				,					"w"       ) as f:
  f.write(_A       )
 return filename



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 import bza

 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.bz2"
 __a							=	bytes(_A				,					"utf-8"       )
 with bza.open(_A				,					"wb"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 import gzip

 __a							=	str(tmp_path_factory.mktemp("data"       ) / "file.txt.gz"       )
 __a							=	bytes(_A				,					"utf-8"       )
 with gzip.open(_A				,					"wb"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 if datasets.config.LZ4_AVAILABLE:
  import lza.frame

  __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.lz4"
  __a							=	bytes(_A				,					"utf-8"       )
  with lza.frame.open(_A				,					"wb"       ) as f:
   f.write(_A       )
  return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 if datasets.config.PY7ZR_AVAILABLE:
  import pyazr

  __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.7z"
  with pyazr.SevenZipFile(_A				,					"w"       ) as archive:
   archive.write(_A				,					arcname=os.path.basename(_A       )       )
  return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import tarfile

 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.tar"
 with tarfile.TarFile(_A				,					"w"       ) as f:
  f.add(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 import lzma

 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.xz"
 __a							=	bytes(_A				,					"utf-8"       )
 with lzma.open(_A				,					"wb"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import zipfile

 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 if datasets.config.ZSTANDARD_AVAILABLE:
  import zstandard as zstd

  __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.zst"
  __a							=	bytes(_A				,					"utf-8"       )
  with zstd.open(_A				,					"wb"       ) as f:
   f.write(_A       )
  return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "file.xml"
 __a							=	textwrap.dedent(
     "\\n    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n    <tmx version=\"1.4\">\n      <header segtype=\"sentence\" srclang=\"ca\" />\n      <body>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>"       )
 with open(_A				,					"w"       ) as f:
  f.write(_A       )
 return filename


SCREAMING_SNAKE_CASE			:						Any				  =		[
    {"""col_1""": """0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """3""", """col_2""": 3, """col_3""": 3.0},
]
SCREAMING_SNAKE_CASE			:						Optional[int]				  =		[
    {"""col_1""": """4""", """col_2""": 4, """col_3""": 4.0},
    {"""col_1""": """5""", """col_2""": 5, """col_3""": 5.0},
]
SCREAMING_SNAKE_CASE			:						Any				  =		{
    """col_1""": ["""0""", """1""", """2""", """3"""],
    """col_2""": [0, 1, 2, 3],
    """col_3""": [0.0, 1.0, 2.0, 3.0],
}

SCREAMING_SNAKE_CASE			:						int				  =		[
    {"""col_3""": 0.0, """col_1""": """0""", """col_2""": 0},
    {"""col_3""": 1.0, """col_1""": """1""", """col_2""": 1},
]

SCREAMING_SNAKE_CASE			:						List[Any]				  =		[
    {"""col_1""": """s0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """s1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """s2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """s3""", """col_2""": 3, """col_3""": 3.0},
]



@pytest.fixture(scope="session"       )
def 	__A		(      ):



 """simple docstring"""


 return DATA_DICT_OF_LISTS



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	datasets.Dataset.from_dict(_A       )
 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.arrow"       )
 dataset.map(cache_file_name=_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.sqlite"       )
 with contextlib.closing(sqlitea.connect(_A       )       ) as con:
  __a							=	con.cursor()
  cur.execute("CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)"       )
  for item in DATA:
   cur.execute("INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)"				,					tuple(item.values()       )       )
  con.commit()
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.csv"       )
 with open(_A				,					"w"				,					newline=""       ) as f:
  __a							=	csv.DictWriter(_A				,					fieldnames=["col_1", "col_2", "col_3"]       )
  writer.writeheader()
  for item in DATA:
   writer.writerow(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset2.csv"       )
 with open(_A				,					"w"				,					newline=""       ) as f:
  __a							=	csv.DictWriter(_A				,					fieldnames=["col_1", "col_2", "col_3"]       )
  writer.writeheader()
  for item in DATA:
   writer.writerow(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import bza

 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.csv.bz2"
 with open(_A				,					"rb"       ) as f:
  __a							=	f.read()
 # data = bytes(FILE_CONTENT, "utf-8")
 with bza.open(_A				,					"wb"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.csv.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
  f.write(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.csv.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(csv_path.replace(".csv"				,					".CSV"       )       )       )
  f.write(_A				,					arcname=os.path.basename(csva_path.replace(".csv"				,					".CSV"       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_with_dir.csv.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.parquet"       )
 __a							=	pa.schema(
     {
         "col_1": pa.string(),
         "col_2": pa.intaa(),
         "col_3": pa.floataa(),
     }       )
 with open(_A				,					"wb"       ) as f:
  __a							=	pq.ParquetWriter(_A				,					schema=_A       )
  __a							=	pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(_A       )       )] for k in DATA[0]}				,					schema=_A       )
  writer.write_table(_A       )
  writer.close()
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.json"       )
 __a							=	{"data": DATA}
 with open(_A				,					"w"       ) as f:
  json.dump(_A				,					_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.json"       )
 __a							=	{"data": DATA_DICT_OF_LISTS}
 with open(_A				,					"w"       ) as f:
  json.dump(_A				,					_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.jsonl"       )
 with open(_A				,					"w"       ) as f:
  for item in DATA:
   f.write(json.dumps(_A       ) + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset2.jsonl"       )
 with open(_A				,					"w"       ) as f:
  for item in DATA:
   f.write(json.dumps(_A       ) + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset_312.jsonl"       )
 with open(_A				,					"w"       ) as f:
  for item in DATA_312:
   f.write(json.dumps(_A       ) + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset-str.jsonl"       )
 with open(_A				,					"w"       ) as f:
  for item in DATA_STR:
   f.write(json.dumps(_A       ) + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import gzip

 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.txt.gz"       )
 with open(_A				,					"rb"       ) as orig_file:
  with gzip.open(_A				,					"wb"       ) as zipped_file:
   zipped_file.writelines(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import gzip

 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.jsonl.gz"       )
 with open(_A				,					"rb"       ) as orig_file:
  with gzip.open(_A				,					"wb"       ) as zipped_file:
   zipped_file.writelines(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.jsonl.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
  f.write(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_nested.jsonl.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.join("nested"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_with_dir.jsonl.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.jsonl.tar"
 with tarfile.TarFile(_A				,					"w"       ) as f:
  f.add(_A				,					arcname=os.path.basename(_A       )       )
  f.add(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_nested.jsonl.tar"
 with tarfile.TarFile(_A				,					"w"       ) as f:
  f.add(_A				,					arcname=os.path.join("nested"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	["0", "1", "2", "3"]
 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.txt"       )
 with open(_A				,					"w"       ) as f:
  for item in data:
   f.write(item + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	["0", "1", "2", "3"]
 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset2.txt"       )
 with open(_A				,					"w"       ) as f:
  for item in data:
   f.write(item + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	["0", "1", "2", "3"]
 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.abc"
 with open(_A				,					"w"       ) as f:
  for item in data:
   f.write(item + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.text.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
  f.write(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_with_dir.text.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.ext.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename("unsupported.ext"       )       )
  f.write(_A				,					arcname=os.path.basename("unsupported_2.ext"       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	"\n".join(["First", "Second\u2029with Unicode new line", "Third"]       )
 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset_with_unicode_new_lines.txt"       )
 with open(_A				,					"w"				,					encoding="utf-8"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      ):



 """simple docstring"""


 return os.path.join("tests"				,					"features"				,					"data"				,					"test_image_rgb.jpg"       )



@pytest.fixture(scope="session"       )
def 	__A		(      ):



 """simple docstring"""


 return os.path.join("tests"				,					"features"				,					"data"				,					"test_audio_44100.wav"       )



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.img.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
  f.write(_A				,					arcname=os.path.basename(_A       ).replace(".jpg"				,					"2.jpg"       )       )
 return path






@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data_dir"       )

 (data_dir / "subdir").mkdir()
 with open(data_dir / "subdir" / "train.txt"				,					"w"       ) as f:
  f.write("foo\n" * 10       )
 with open(data_dir / "subdir" / "test.txt"				,					"w"       ) as f:
  f.write("bar\n" * 10       )
 # hidden file
 with open(data_dir / "subdir" / ".test.txt"				,					"w"       ) as f:
  f.write("bar\n" * 10       )

 # hidden directory
 (data_dir / ".subdir").mkdir()
 with open(data_dir / ".subdir" / "train.txt"				,					"w"       ) as f:
  f.write("foo\n" * 10       )
 with open(data_dir / ".subdir" / "test.txt"				,					"w"       ) as f:
  f.write("bar\n" * 10       )

 return data_dir


