import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import BlenderbotConfig, is_flax_available
from transformers.testing_utils import jax_device, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
  import os

  # The slow tests are often failing with OOM error on GPU
  # This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
  # but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
  SCREAMING_SNAKE_CASE			:						int				  =		"""platform"""

  import jax
  import jax.numpy as jnp

  from transformers import BlenderbotTokenizer
  from transformers.models.blenderbot.modeling_flax_blenderbot import (
      FlaxBlenderbotForConditionalGeneration,
      FlaxBlenderbotModel,
      shift_tokens_right,
  )



def 	__A		(      _A				,					_A				,					_A=None				,					_A=None				,					_A=None				,					_A=None				,					_A=None				,					_A=None				,					):



 """simple docstring"""


 if attention_mask is None:
  __a							=	np.where(input_ids != config.pad_token_id				,					1				,					0       )
 if decoder_attention_mask is None:
  __a							=	np.where(decoder_input_ids != config.pad_token_id				,					1				,					0       )
 if head_mask is None:
  __a							=	np.ones((config.encoder_layers, config.encoder_attention_heads)       )
 if decoder_head_mask is None:
  __a							=	np.ones((config.decoder_layers, config.decoder_attention_heads)       )
 if cross_attn_head_mask is None:
  __a							=	np.ones((config.decoder_layers, config.decoder_attention_heads)       )
 return {
     "input_ids": input_ids,
     "decoder_input_ids": decoder_input_ids,
     "attention_mask": attention_mask,
     "decoder_attention_mask": attention_mask,
 }
class  A_			:

  def __init__(		self			:   Any  ,				__SCREAMING_SNAKE_CASE			:   List[Any]  ,				__SCREAMING_SNAKE_CASE			:   List[Any]=13  ,				__SCREAMING_SNAKE_CASE			:   Optional[Any]=7  ,				__SCREAMING_SNAKE_CASE			:   Union[str, Any]=True  ,				__SCREAMING_SNAKE_CASE			:   Dict=False  ,				__SCREAMING_SNAKE_CASE			:   int=99  ,				__SCREAMING_SNAKE_CASE			:   int=16  ,				__SCREAMING_SNAKE_CASE			:   List[Any]=2  ,				__SCREAMING_SNAKE_CASE			:   int=4  ,				__SCREAMING_SNAKE_CASE			:   Optional[Any]=4  ,				__SCREAMING_SNAKE_CASE			:   str="gelu"  ,				__SCREAMING_SNAKE_CASE			:   List[str]=0.1  ,				__SCREAMING_SNAKE_CASE			:   Tuple=0.1  ,				__SCREAMING_SNAKE_CASE			:   Tuple=32  ,				__SCREAMING_SNAKE_CASE			:   Optional[Any]=2  ,				__SCREAMING_SNAKE_CASE			:   Any=1  ,				__SCREAMING_SNAKE_CASE			:   Optional[Any]=0  ,				__SCREAMING_SNAKE_CASE			:   int=0.02  ,				):
   __a							=	parent
   __a							=	batch_size
   __a							=	seq_length
   __a							=	is_training
   __a							=	use_labels
   __a							=	vocab_size
   __a							=	hidden_size
   __a							=	num_hidden_layers
   __a							=	num_attention_heads
   __a							=	intermediate_size
   __a							=	hidden_act
   __a							=	hidden_dropout_prob
   __a							=	attention_probs_dropout_prob
   __a							=	max_position_embeddings
   __a							=	eos_token_id
   __a							=	pad_token_id
   __a							=	bos_token_id
   __a							=	initializer_range

  def 		_UpperCAmelCase (		self			:   Optional[Any]							):
   __a							=	np.clip(ids_tensor([self.batch_size, self.seq_length - 1]  ,				self.vocab_size							)  ,				3  ,				self.vocab_size							)
   __a							=	np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)  ,				dtype=np.intaa							))  ,				-1							)

   __a							=	shift_tokens_right(__SCREAMING_SNAKE_CASE  ,				1  ,				2							)

   __a							=	BlenderbotConfig(
       vocab_size=self.vocab_size  ,				d_model=self.hidden_size  ,				encoder_layers=self.num_hidden_layers  ,				decoder_layers=self.num_hidden_layers  ,				encoder_attention_heads=self.num_attention_heads  ,				decoder_attention_heads=self.num_attention_heads  ,				encoder_ffn_dim=self.intermediate_size  ,				decoder_ffn_dim=self.intermediate_size  ,				dropout=self.hidden_dropout_prob  ,				attention_dropout=self.attention_probs_dropout_prob  ,				max_position_embeddings=self.max_position_embeddings  ,				eos_token_id=self.eos_token_id  ,				bos_token_id=self.bos_token_id  ,				pad_token_id=self.pad_token_id  ,				initializer_range=self.initializer_range  ,				use_cache=__SCREAMING_SNAKE_CASE  ,				)
   __a							=	prepare_blenderbot_inputs_dict(__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE							)
   return config, inputs_dict

  def 		_UpperCAmelCase (		self			:   Tuple							):
   __a							,  __a							=	self.prepare_config_and_inputs()
   return config, inputs_dict

  def 		_UpperCAmelCase (		self			:   Optional[int]  ,				__SCREAMING_SNAKE_CASE			:   Tuple  ,				__SCREAMING_SNAKE_CASE			:   int  ,				__SCREAMING_SNAKE_CASE			:   int							):
   __a							=	20
   __a							=	model_class_name(__SCREAMING_SNAKE_CASE							)

   __a							=	model.encode(inputs_dict["input_ids"]							)

   __a							,  __a							=	(
       inputs_dict["decoder_input_ids"],
       inputs_dict["decoder_attention_mask"],
   )

   __a							=	model.init_cache(decoder_input_ids.shape[0]  ,				__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE							)
   __a							=	jnp.ones((decoder_input_ids.shape[0], max_decoder_length)  ,				dtype="i4"							)

   __a							=	jnp.broadcast_to(
       jnp.arange(decoder_input_ids.shape[-1] - 1							)[None, :]  ,				(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)  ,				)
   __a							=	model.decode(
       decoder_input_ids[:, :-1]  ,				__SCREAMING_SNAKE_CASE  ,				decoder_attention_mask=__SCREAMING_SNAKE_CASE  ,				past_key_values=__SCREAMING_SNAKE_CASE  ,				decoder_position_ids=__SCREAMING_SNAKE_CASE  ,				)

   __a							=	jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]  ,				dtype="i4"							)
   __a							=	model.decode(
       decoder_input_ids[:, -1:]  ,				__SCREAMING_SNAKE_CASE  ,				decoder_attention_mask=__SCREAMING_SNAKE_CASE  ,				past_key_values=outputs_cache.past_key_values  ,				decoder_position_ids=__SCREAMING_SNAKE_CASE  ,				)

   __a							=	model.decode(__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE							)

   __a							=	np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])							)							)
   self.parent.assertTrue(diff < 1E-3  ,				msg=f"""Max diff is {diff}"""							)

  def 		_UpperCAmelCase (		self			:   str  ,				__SCREAMING_SNAKE_CASE			:   List[Any]  ,				__SCREAMING_SNAKE_CASE			:   Union[str, Any]  ,				__SCREAMING_SNAKE_CASE			:   Dict							):
   __a							=	20
   __a							=	model_class_name(__SCREAMING_SNAKE_CASE							)

   __a							=	model.encode(inputs_dict["input_ids"]							)

   __a							,  __a							=	(
       inputs_dict["decoder_input_ids"],
       inputs_dict["decoder_attention_mask"],
   )

   __a							=	jnp.concatenate(
       [
           decoder_attention_mask,
           jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])							),
       ]  ,				axis=-1  ,				)

   __a							=	model.init_cache(decoder_input_ids.shape[0]  ,				__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE							)
   __a							=	jnp.broadcast_to(
       jnp.arange(decoder_input_ids.shape[-1] - 1							)[None, :]  ,				(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)  ,				)

   __a							=	model.decode(
       decoder_input_ids[:, :-1]  ,				__SCREAMING_SNAKE_CASE  ,				decoder_attention_mask=__SCREAMING_SNAKE_CASE  ,				past_key_values=__SCREAMING_SNAKE_CASE  ,				decoder_position_ids=__SCREAMING_SNAKE_CASE  ,				)
   __a							=	jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]  ,				dtype="i4"							)
   __a							=	model.decode(
       decoder_input_ids[:, -1:]  ,				__SCREAMING_SNAKE_CASE  ,				past_key_values=outputs_cache.past_key_values  ,				decoder_attention_mask=__SCREAMING_SNAKE_CASE  ,				decoder_position_ids=__SCREAMING_SNAKE_CASE  ,				)

   __a							=	model.decode(__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE  ,				decoder_attention_mask=__SCREAMING_SNAKE_CASE							)

   __a							=	np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])							)							)
   self.parent.assertTrue(diff < 1E-3  ,				msg=f"""Max diff is {diff}"""							)
@require_flax
class  A_			(							unittest.TestCase			):
  _SCREAMING_SNAKE_CASE     					=      99

  def 		_UpperCAmelCase (		self			:   List[Any]							):
   __a							=	np.array(
       [
           [71, 82, 18, 33, 46, 91, 2],
           [68, 34, 26, 58, 30, 82, 2],
           [5, 97, 17, 39, 94, 40, 2],
           [76, 83, 94, 25, 70, 78, 2],
           [87, 59, 41, 35, 48, 66, 2],
           [55, 13, 16, 58, 5, 2, 1],  # note padding
           [64, 27, 31, 51, 12, 75, 2],
           [52, 64, 86, 17, 83, 39, 2],
           [48, 61, 9, 24, 71, 82, 2],
           [26, 1, 60, 48, 22, 13, 2],
           [21, 5, 62, 28, 14, 76, 2],
           [45, 98, 37, 86, 59, 48, 2],
           [70, 70, 50, 9, 28, 0, 2],
       ]  ,				dtype=np.intaa  ,				)

   __a							=	input_ids.shape[0]
   __a							=	BlenderbotConfig(
       vocab_size=self.vocab_size  ,				d_model=24  ,				encoder_layers=2  ,				decoder_layers=2  ,				encoder_attention_heads=2  ,				decoder_attention_heads=2  ,				encoder_ffn_dim=32  ,				decoder_ffn_dim=32  ,				max_position_embeddings=48  ,				eos_token_id=2  ,				pad_token_id=1  ,				bos_token_id=0  ,				)
   return config, input_ids, batch_size

  def 		_UpperCAmelCase (		self			:   str							):
   __a							,  __a							,  __a							=	self._get_config_and_data()
   __a							=	FlaxBlenderbotForConditionalGeneration(__SCREAMING_SNAKE_CASE							)
   __a							=	lm_model(input_ids=__SCREAMING_SNAKE_CASE							)
   __a							=	(batch_size, input_ids.shape[1], config.vocab_size)
   self.assertEqual(outputs["logits"].shape  ,				__SCREAMING_SNAKE_CASE							)

  def 		_UpperCAmelCase (		self			:   Dict							):
   __a							=	BlenderbotConfig(
       vocab_size=self.vocab_size  ,				d_model=14  ,				encoder_layers=2  ,				decoder_layers=2  ,				encoder_attention_heads=2  ,				decoder_attention_heads=2  ,				encoder_ffn_dim=8  ,				decoder_ffn_dim=8  ,				max_position_embeddings=48  ,				)
   __a							=	FlaxBlenderbotForConditionalGeneration(__SCREAMING_SNAKE_CASE							)
   __a							=	np.array([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]]  ,				dtype=np.intaa							)
   __a							=	np.array([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]]  ,				dtype=np.intaa							)
   __a							=	lm_model(input_ids=__SCREAMING_SNAKE_CASE  ,				decoder_input_ids=__SCREAMING_SNAKE_CASE							)
   __a							=	(*summary.shape, config.vocab_size)
   self.assertEqual(outputs["logits"].shape  ,				__SCREAMING_SNAKE_CASE							)

  def 		_UpperCAmelCase (		self			:   Any							):
   __a							=	np.array([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]]  ,				dtype=np.intaa							)
   __a							=	shift_tokens_right(__SCREAMING_SNAKE_CASE  ,				1  ,				2							)
   __a							=	np.equal(__SCREAMING_SNAKE_CASE  ,				1							).astype(np.floataa							).sum()
   __a							=	np.equal(__SCREAMING_SNAKE_CASE  ,				1							).astype(np.floataa							).sum()
   self.assertEqual(shifted.shape  ,				input_ids.shape							)
   self.assertEqual(__SCREAMING_SNAKE_CASE  ,				n_pad_before - 1							)
   self.assertTrue(np.equal(shifted[:, 0]  ,				2							).all()							)
@require_flax
class  A_			(							a_      ,							unittest.TestCase      ,							a_			):
  _SCREAMING_SNAKE_CASE     					=      True
  _SCREAMING_SNAKE_CASE     					=      (
      (
          FlaxBlenderbotModel,
          FlaxBlenderbotForConditionalGeneration,
      )
      if is_flax_available()
      else ()
  )
  _SCREAMING_SNAKE_CASE     					=      (FlaxBlenderbotForConditionalGeneration,) if is_flax_available() else ()

  def 		_UpperCAmelCase (		self			:   Any							):
   __a							=	FlaxBlenderbotModelTester(self							)

  def 		_UpperCAmelCase (		self			:   str							):
   __a							,  __a							=	self.model_tester.prepare_config_and_inputs()
   for model_class in self.all_model_classes:
    self.model_tester.check_use_cache_forward(__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE							)

  def 		_UpperCAmelCase (		self			:   Optional[int]							):
   __a							,  __a							=	self.model_tester.prepare_config_and_inputs()
   for model_class in self.all_model_classes:
    self.model_tester.check_use_cache_forward_with_attn_mask(__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE							)

  def 		_UpperCAmelCase (		self			:   Optional[Any]							):
   __a							,  __a							=	self.model_tester.prepare_config_and_inputs_for_common()

   for model_class in self.all_model_classes:
    with self.subTest(model_class.__name__							):
     __a							=	self._prepare_for_class(__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE							)
     __a							=	model_class(__SCREAMING_SNAKE_CASE							)

     @jax.jit
     def encode_jitted(__SCREAMING_SNAKE_CASE			:   Tuple  ,				__SCREAMING_SNAKE_CASE			:   List[str]=None  ,				**__SCREAMING_SNAKE_CASE			:   Optional[int]							):
      return model.encode(input_ids=__SCREAMING_SNAKE_CASE  ,				attention_mask=__SCREAMING_SNAKE_CASE							)

     with self.subTest("JIT Enabled"							):
      __a							=	encode_jitted(**__SCREAMING_SNAKE_CASE							).to_tuple()

     with self.subTest("JIT Disabled"							):
      with jax.disable_jit():
       __a							=	encode_jitted(**__SCREAMING_SNAKE_CASE							).to_tuple()

     self.assertEqual(len(__SCREAMING_SNAKE_CASE							)  ,				len(__SCREAMING_SNAKE_CASE							)							)
     for jitted_output, output in zip(__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE							):
      self.assertEqual(jitted_output.shape  ,				output.shape							)

  def 		_UpperCAmelCase (		self			:   List[Any]							):
   __a							,  __a							=	self.model_tester.prepare_config_and_inputs_for_common()

   for model_class in self.all_model_classes:
    with self.subTest(model_class.__name__							):
     __a							=	model_class(__SCREAMING_SNAKE_CASE							)
     __a							=	model.encode(inputs_dict["input_ids"]  ,				inputs_dict["attention_mask"]							)

     __a							=	{
         "decoder_input_ids": inputs_dict["decoder_input_ids"],
         "decoder_attention_mask": inputs_dict["decoder_attention_mask"],
         "encoder_outputs": encoder_outputs,
     }

     @jax.jit
     def decode_jitted(__SCREAMING_SNAKE_CASE			:   Any  ,				__SCREAMING_SNAKE_CASE			:   Any  ,				__SCREAMING_SNAKE_CASE			:   Any							):
      return model.decode(
          decoder_input_ids=__SCREAMING_SNAKE_CASE  ,				decoder_attention_mask=__SCREAMING_SNAKE_CASE  ,				encoder_outputs=__SCREAMING_SNAKE_CASE  ,				)

     with self.subTest("JIT Enabled"							):
      __a							=	decode_jitted(**__SCREAMING_SNAKE_CASE							).to_tuple()

     with self.subTest("JIT Disabled"							):
      with jax.disable_jit():
       __a							=	decode_jitted(**__SCREAMING_SNAKE_CASE							).to_tuple()

     self.assertEqual(len(__SCREAMING_SNAKE_CASE							)  ,				len(__SCREAMING_SNAKE_CASE							)							)
     for jitted_output, output in zip(__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE							):
      self.assertEqual(jitted_output.shape  ,				output.shape							)

  @slow
  def 		_UpperCAmelCase (		self			:   Tuple							):
   for model_class_name in self.all_model_classes:
    __a							=	model_class_name.from_pretrained("facebook/blenderbot-400M-distill"							)
    # FlaxBlenderbotForSequenceClassification expects eos token in input_ids
    __a							=	np.ones((1, 1)							) * model.config.eos_token_id
    __a							=	model(__SCREAMING_SNAKE_CASE							)
    self.assertIsNotNone(__SCREAMING_SNAKE_CASE							)

  @unittest.skipUnless(jax_device != "cpu"  ,				"3B test too slow on CPU."							)
  @slow
  def 		_UpperCAmelCase (		self			:   int							):
   __a							=	{"num_beams": 1, "early_stopping": True, "min_length": 15, "max_length": 25}
   __a							=	{"skip_special_tokens": True, "clean_up_tokenization_spaces": True}

   __a							=	FlaxBlenderbotForConditionalGeneration.from_pretrained("facebook/blenderbot-3B"  ,				from_pt=__SCREAMING_SNAKE_CASE							)
   __a							=	BlenderbotTokenizer.from_pretrained("facebook/blenderbot-3B"							)

   __a							=	["Sam"]
   __a							=	tokenizer(__SCREAMING_SNAKE_CASE  ,				return_tensors="jax"							)

   __a							=	model.generate(**__SCREAMING_SNAKE_CASE  ,				**__SCREAMING_SNAKE_CASE							)
   __a							=	"Sam is a great name. It means \"sun\" in Gaelic."

   __a							=	tokenizer.batch_decode(__SCREAMING_SNAKE_CASE  ,				**__SCREAMING_SNAKE_CASE							)
   assert generated_txt[0].strip() == tgt_text



import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config



@pytest.fixture(scope="session"       )
def 	__A		(      ):



 """simple docstring"""


 __a							=	10
 __a							=	datasets.Features(
     {
         "tokens": datasets.Sequence(datasets.Value("string"       )       ),
         "labels": datasets.Sequence(datasets.ClassLabel(names=["negative", "positive"]       )       ),
         "answers": datasets.Sequence(
             {
                 "text": datasets.Value("string"       ),
                 "answer_start": datasets.Value("int32"       ),
             }       ),
         "id": datasets.Value("int64"       ),
     }       )
 __a							=	datasets.Dataset.from_dict(
     {
         "tokens": [["foo"] * 5] * n,
         "labels": [[1] * 5] * n,
         "answers": [{"answer_start": [97], "text": ["1976"]}] * 10,
         "id": list(range(_A       )       ),
     }				,					features=_A				,					)
 return dataset



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "file.arrow"       )
 dataset.map(cache_file_name=_A       )
 return filename


# FILE_CONTENT + files


SCREAMING_SNAKE_CASE			:						Tuple				  =		"""\
    Text data.
    Second line of data."""



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt"
 __a							=	FILE_CONTENT
 with open(_A				,					"w"       ) as f:
  f.write(_A       )
 return filename



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 import bza

 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.bz2"
 __a							=	bytes(_A				,					"utf-8"       )
 with bza.open(_A				,					"wb"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 import gzip

 __a							=	str(tmp_path_factory.mktemp("data"       ) / "file.txt.gz"       )
 __a							=	bytes(_A				,					"utf-8"       )
 with gzip.open(_A				,					"wb"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 if datasets.config.LZ4_AVAILABLE:
  import lza.frame

  __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.lz4"
  __a							=	bytes(_A				,					"utf-8"       )
  with lza.frame.open(_A				,					"wb"       ) as f:
   f.write(_A       )
  return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 if datasets.config.PY7ZR_AVAILABLE:
  import pyazr

  __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.7z"
  with pyazr.SevenZipFile(_A				,					"w"       ) as archive:
   archive.write(_A				,					arcname=os.path.basename(_A       )       )
  return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import tarfile

 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.tar"
 with tarfile.TarFile(_A				,					"w"       ) as f:
  f.add(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 import lzma

 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.xz"
 __a							=	bytes(_A				,					"utf-8"       )
 with lzma.open(_A				,					"wb"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import zipfile

 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 if datasets.config.ZSTANDARD_AVAILABLE:
  import zstandard as zstd

  __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.zst"
  __a							=	bytes(_A				,					"utf-8"       )
  with zstd.open(_A				,					"wb"       ) as f:
   f.write(_A       )
  return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "file.xml"
 __a							=	textwrap.dedent(
     "\\n    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n    <tmx version=\"1.4\">\n      <header segtype=\"sentence\" srclang=\"ca\" />\n      <body>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>"       )
 with open(_A				,					"w"       ) as f:
  f.write(_A       )
 return filename


SCREAMING_SNAKE_CASE			:						Any				  =		[
    {"""col_1""": """0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """3""", """col_2""": 3, """col_3""": 3.0},
]
SCREAMING_SNAKE_CASE			:						Optional[int]				  =		[
    {"""col_1""": """4""", """col_2""": 4, """col_3""": 4.0},
    {"""col_1""": """5""", """col_2""": 5, """col_3""": 5.0},
]
SCREAMING_SNAKE_CASE			:						Any				  =		{
    """col_1""": ["""0""", """1""", """2""", """3"""],
    """col_2""": [0, 1, 2, 3],
    """col_3""": [0.0, 1.0, 2.0, 3.0],
}

SCREAMING_SNAKE_CASE			:						int				  =		[
    {"""col_3""": 0.0, """col_1""": """0""", """col_2""": 0},
    {"""col_3""": 1.0, """col_1""": """1""", """col_2""": 1},
]

SCREAMING_SNAKE_CASE			:						List[Any]				  =		[
    {"""col_1""": """s0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """s1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """s2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """s3""", """col_2""": 3, """col_3""": 3.0},
]



@pytest.fixture(scope="session"       )
def 	__A		(      ):



 """simple docstring"""


 return DATA_DICT_OF_LISTS



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	datasets.Dataset.from_dict(_A       )
 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.arrow"       )
 dataset.map(cache_file_name=_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.sqlite"       )
 with contextlib.closing(sqlitea.connect(_A       )       ) as con:
  __a							=	con.cursor()
  cur.execute("CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)"       )
  for item in DATA:
   cur.execute("INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)"				,					tuple(item.values()       )       )
  con.commit()
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.csv"       )
 with open(_A				,					"w"				,					newline=""       ) as f:
  __a							=	csv.DictWriter(_A				,					fieldnames=["col_1", "col_2", "col_3"]       )
  writer.writeheader()
  for item in DATA:
   writer.writerow(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset2.csv"       )
 with open(_A				,					"w"				,					newline=""       ) as f:
  __a							=	csv.DictWriter(_A				,					fieldnames=["col_1", "col_2", "col_3"]       )
  writer.writeheader()
  for item in DATA:
   writer.writerow(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import bza

 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.csv.bz2"
 with open(_A				,					"rb"       ) as f:
  __a							=	f.read()
 # data = bytes(FILE_CONTENT, "utf-8")
 with bza.open(_A				,					"wb"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.csv.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
  f.write(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.csv.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(csv_path.replace(".csv"				,					".CSV"       )       )       )
  f.write(_A				,					arcname=os.path.basename(csva_path.replace(".csv"				,					".CSV"       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_with_dir.csv.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.parquet"       )
 __a							=	pa.schema(
     {
         "col_1": pa.string(),
         "col_2": pa.intaa(),
         "col_3": pa.floataa(),
     }       )
 with open(_A				,					"wb"       ) as f:
  __a							=	pq.ParquetWriter(_A				,					schema=_A       )
  __a							=	pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(_A       )       )] for k in DATA[0]}				,					schema=_A       )
  writer.write_table(_A       )
  writer.close()
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.json"       )
 __a							=	{"data": DATA}
 with open(_A				,					"w"       ) as f:
  json.dump(_A				,					_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.json"       )
 __a							=	{"data": DATA_DICT_OF_LISTS}
 with open(_A				,					"w"       ) as f:
  json.dump(_A				,					_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.jsonl"       )
 with open(_A				,					"w"       ) as f:
  for item in DATA:
   f.write(json.dumps(_A       ) + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset2.jsonl"       )
 with open(_A				,					"w"       ) as f:
  for item in DATA:
   f.write(json.dumps(_A       ) + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset_312.jsonl"       )
 with open(_A				,					"w"       ) as f:
  for item in DATA_312:
   f.write(json.dumps(_A       ) + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset-str.jsonl"       )
 with open(_A				,					"w"       ) as f:
  for item in DATA_STR:
   f.write(json.dumps(_A       ) + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import gzip

 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.txt.gz"       )
 with open(_A				,					"rb"       ) as orig_file:
  with gzip.open(_A				,					"wb"       ) as zipped_file:
   zipped_file.writelines(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import gzip

 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.jsonl.gz"       )
 with open(_A				,					"rb"       ) as orig_file:
  with gzip.open(_A				,					"wb"       ) as zipped_file:
   zipped_file.writelines(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.jsonl.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
  f.write(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_nested.jsonl.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.join("nested"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_with_dir.jsonl.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.jsonl.tar"
 with tarfile.TarFile(_A				,					"w"       ) as f:
  f.add(_A				,					arcname=os.path.basename(_A       )       )
  f.add(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_nested.jsonl.tar"
 with tarfile.TarFile(_A				,					"w"       ) as f:
  f.add(_A				,					arcname=os.path.join("nested"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	["0", "1", "2", "3"]
 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.txt"       )
 with open(_A				,					"w"       ) as f:
  for item in data:
   f.write(item + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	["0", "1", "2", "3"]
 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset2.txt"       )
 with open(_A				,					"w"       ) as f:
  for item in data:
   f.write(item + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	["0", "1", "2", "3"]
 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.abc"
 with open(_A				,					"w"       ) as f:
  for item in data:
   f.write(item + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.text.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
  f.write(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_with_dir.text.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.ext.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename("unsupported.ext"       )       )
  f.write(_A				,					arcname=os.path.basename("unsupported_2.ext"       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	"\n".join(["First", "Second\u2029with Unicode new line", "Third"]       )
 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset_with_unicode_new_lines.txt"       )
 with open(_A				,					"w"				,					encoding="utf-8"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      ):



 """simple docstring"""


 return os.path.join("tests"				,					"features"				,					"data"				,					"test_image_rgb.jpg"       )



@pytest.fixture(scope="session"       )
def 	__A		(      ):



 """simple docstring"""


 return os.path.join("tests"				,					"features"				,					"data"				,					"test_audio_44100.wav"       )



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.img.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
  f.write(_A				,					arcname=os.path.basename(_A       ).replace(".jpg"				,					"2.jpg"       )       )
 return path






@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data_dir"       )

 (data_dir / "subdir").mkdir()
 with open(data_dir / "subdir" / "train.txt"				,					"w"       ) as f:
  f.write("foo\n" * 10       )
 with open(data_dir / "subdir" / "test.txt"				,					"w"       ) as f:
  f.write("bar\n" * 10       )
 # hidden file
 with open(data_dir / "subdir" / ".test.txt"				,					"w"       ) as f:
  f.write("bar\n" * 10       )

 # hidden directory
 (data_dir / ".subdir").mkdir()
 with open(data_dir / ".subdir" / "train.txt"				,					"w"       ) as f:
  f.write("foo\n" * 10       )
 with open(data_dir / ".subdir" / "test.txt"				,					"w"       ) as f:
  f.write("bar\n" * 10       )

 return data_dir


