



import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       XLMForMultipleChoice,
       XLMForQuestionAnswering,
       XLMForQuestionAnsweringSimple,
       XLMForSequenceClassification,
       XLMForTokenClassification,
       XLMModel,
       XLMWithLMHeadModel,
   )
   from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST



class     _UpperCAmelCase			:




     def __init__(				self ,						a__ ,						a__=1_3 ,						a__=7 ,						a__=True ,						a__=True ,						a__=True ,						a__=True ,						a__=True ,						a__=False ,						a__=False ,						a__=False ,						a__=2 ,						a__=9_9 ,						a__=0 ,						a__=3_2 ,						a__=5 ,						a__=4 ,						a__=0.1 ,						a__=0.1 ,						a__=5_1_2 ,						a__=2 ,						a__=0.0_2 ,						a__=2 ,						a__=4 ,						a__="last" ,						a__=True ,						a__=None ,						a__=0 ,						):
         A__		 =					parent
         A__		 =					batch_size
         A__		 =					seq_length
         A__		 =					is_training
         A__		 =					use_input_lengths
         A__		 =					use_token_type_ids
         A__		 =					use_labels
         A__		 =					gelu_activation
         A__		 =					sinusoidal_embeddings
         A__		 =					causal
         A__		 =					asm
         A__		 =					n_langs
         A__		 =					vocab_size
         A__		 =					n_special
         A__		 =					hidden_size
         A__		 =					num_hidden_layers
         A__		 =					num_attention_heads
         A__		 =					hidden_dropout_prob
         A__		 =					attention_probs_dropout_prob
         A__		 =					max_position_embeddings
         A__		 =					type_sequence_label_size
         A__		 =					initializer_range
         A__		 =					num_labels
         A__		 =					num_choices
         A__		 =					summary_type
         A__		 =					use_proj
         A__		 =					scope
         A__		 =					bos_token_id




     def 		snake_case_		(				self):
         A__		 =					ids_tensor([self.batch_size, self.seq_length] ,						self.vocab_size)
         A__		 =					random_attention_mask([self.batch_size, self.seq_length])

         A__		 =					None
         if self.use_input_lengths:
             A__		 =					(
                 ids_tensor([self.batch_size] ,						vocab_size=2) + self.seq_length - 2
             )  # small variation of seq_length

         A__		 =					None
         if self.use_token_type_ids:
             A__		 =					ids_tensor([self.batch_size, self.seq_length] ,						self.n_langs)

         A__		 =					None
         A__		 =					None
         A__		 =					None
         if self.use_labels:
             A__		 =					ids_tensor([self.batch_size] ,						self.type_sequence_label_size)
             A__		 =					ids_tensor([self.batch_size, self.seq_length] ,						self.num_labels)
             A__		 =					ids_tensor([self.batch_size] ,						2).float()
             A__		 =					ids_tensor([self.batch_size] ,						self.num_choices)

         A__		 =					self.get_config()

         return (
             config,
             input_ids,
             token_type_ids,
             input_lengths,
             sequence_labels,
             token_labels,
             is_impossible_labels,
             choice_labels,
             input_mask,
         )




     def 		snake_case_		(				self):
         return XLMConfig(
             vocab_size=self.vocab_size ,						n_special=self.n_special ,						emb_dim=self.hidden_size ,						n_layers=self.num_hidden_layers ,						n_heads=self.num_attention_heads ,						dropout=self.hidden_dropout_prob ,						attention_dropout=self.attention_probs_dropout_prob ,						gelu_activation=self.gelu_activation ,						sinusoidal_embeddings=self.sinusoidal_embeddings ,						asm=self.asm ,						causal=self.causal ,						n_langs=self.n_langs ,						max_position_embeddings=self.max_position_embeddings ,						initializer_range=self.initializer_range ,						summary_type=self.summary_type ,						use_proj=self.use_proj ,						num_labels=self.num_labels ,						bos_token_id=self.bos_token_id ,						)




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					XLMModel(config=a__)
         model.to(a__)
         model.eval()
         A__		 =					model(a__ ,						lengths=a__ ,						langs=a__)
         A__		 =					model(a__ ,						langs=a__)
         A__		 =					model(a__)
         self.parent.assertEqual(result.last_hidden_state.shape ,						(self.batch_size, self.seq_length, self.hidden_size))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					XLMWithLMHeadModel(a__)
         model.to(a__)
         model.eval()

         A__		 =					model(a__ ,						token_type_ids=a__ ,						labels=a__)
         self.parent.assertEqual(result.loss.shape ,						())
         self.parent.assertEqual(result.logits.shape ,						(self.batch_size, self.seq_length, self.vocab_size))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					XLMForQuestionAnsweringSimple(a__)
         model.to(a__)
         model.eval()

         A__		 =					model(a__)

         A__		 =					model(a__ ,						start_positions=a__ ,						end_positions=a__)
         A__		 =					outputs
         self.parent.assertEqual(result.start_logits.shape ,						(self.batch_size, self.seq_length))
         self.parent.assertEqual(result.end_logits.shape ,						(self.batch_size, self.seq_length))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					XLMForQuestionAnswering(a__)
         model.to(a__)
         model.eval()

         A__		 =					model(a__)

         A__		 =					model(
             a__ ,						start_positions=a__ ,						end_positions=a__ ,						cls_index=a__ ,						is_impossible=a__ ,						p_mask=a__ ,						)

         A__		 =					model(
             a__ ,						start_positions=a__ ,						end_positions=a__ ,						cls_index=a__ ,						is_impossible=a__ ,						)

         ((A__	) ,		)		 =					result_with_labels.to_tuple()

         A__		 =					model(a__ ,						start_positions=a__ ,						end_positions=a__)

         ((A__	) ,		)		 =					result_with_labels.to_tuple()

         self.parent.assertEqual(result_with_labels.loss.shape ,						())
         self.parent.assertEqual(result.start_top_log_probs.shape ,						(self.batch_size, model.config.start_n_top))
         self.parent.assertEqual(result.start_top_index.shape ,						(self.batch_size, model.config.start_n_top))
         self.parent.assertEqual(
             result.end_top_log_probs.shape ,						(self.batch_size, model.config.start_n_top * model.config.end_n_top))
         self.parent.assertEqual(
             result.end_top_index.shape ,						(self.batch_size, model.config.start_n_top * model.config.end_n_top))
         self.parent.assertEqual(result.cls_logits.shape ,						(self.batch_size,))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					XLMForSequenceClassification(a__)
         model.to(a__)
         model.eval()

         A__		 =					model(a__)
         A__		 =					model(a__ ,						labels=a__)
         self.parent.assertEqual(result.loss.shape ,						())
         self.parent.assertEqual(result.logits.shape ,						(self.batch_size, self.type_sequence_label_size))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					self.num_labels
         A__		 =					XLMForTokenClassification(a__)
         model.to(a__)
         model.eval()

         A__		 =					model(a__ ,						attention_mask=a__ ,						labels=a__)
         self.parent.assertEqual(result.logits.shape ,						(self.batch_size, self.seq_length, self.num_labels))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					self.num_choices
         A__		 =					XLMForMultipleChoice(config=a__)
         model.to(a__)
         model.eval()
         A__		 =					input_ids.unsqueeze(1).expand(-1 ,						self.num_choices ,						-1).contiguous()
         A__		 =					token_type_ids.unsqueeze(1).expand(-1 ,						self.num_choices ,						-1).contiguous()
         A__		 =					input_mask.unsqueeze(1).expand(-1 ,						self.num_choices ,						-1).contiguous()
         A__		 =					model(
             a__ ,						attention_mask=a__ ,						token_type_ids=a__ ,						labels=a__ ,						)
         self.parent.assertEqual(result.logits.shape ,						(self.batch_size, self.num_choices))




     def 		snake_case_		(				self):
         A__		 =					self.prepare_config_and_inputs()
         (
             (
             A__	
         ) ,	(
             A__	
         ) ,	(
             A__	
         ) ,	(
             A__	
         ) ,	(
             A__	
         ) ,	(
             A__	
         ) ,	(
             A__	
         ) ,	(
             A__	
         ) ,	(
             A__	
         ) ,		
         )		 =					config_and_inputs
         A__		 =					{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''lengths''': input_lengths}
         return config, inputs_dict







@require_torch
class     _UpperCAmelCase			(       A__			,      A__			,      A__			,      unittest.TestCase     ):
     UpperCamelCase__		    =				(
         (
             XLMModel,
             XLMWithLMHeadModel,
             XLMForQuestionAnswering,
             XLMForSequenceClassification,
             XLMForQuestionAnsweringSimple,
             XLMForTokenClassification,
             XLMForMultipleChoice,
         )
         if is_torch_available()
         else ()
     )
     UpperCamelCase__		    =				(
         (XLMWithLMHeadModel,) if is_torch_available() else ()
     )  # TODO (PVP): Check other models whether language generation is also applicable
     UpperCamelCase__		    =				(
         {
             '''feature-extraction''': XLMModel,
             '''fill-mask''': XLMWithLMHeadModel,
             '''question-answering''': XLMForQuestionAnsweringSimple,
             '''text-classification''': XLMForSequenceClassification,
             '''text-generation''': XLMWithLMHeadModel,
             '''token-classification''': XLMForTokenClassification,
             '''zero-shot''': XLMForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__):
         if (
             pipeline_test_casse_name == "QAPipelineTests"
             and tokenizer_name is not None
             and not tokenizer_name.endswith('''Fast''')
         ):
             # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
             # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
             # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
             return True

         return False




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__=False):
         A__		 =					super()._prepare_for_class(a__ ,						a__ ,						return_labels=a__)

         if return_labels:
             if model_class.__name__ == "XLMForQuestionAnswering":
                 A__		 =					torch.zeros(
                     self.model_tester.batch_size ,						dtype=torch.long ,						device=a__)
                 A__		 =					torch.zeros(
                     self.model_tester.batch_size ,						dtype=torch.long ,						device=a__)

         return inputs_dict




     def 		snake_case_		(				self):
         A__		 =					XLMModelTester(self)
         A__		 =					ConfigTester(self ,						config_class=a__ ,						emb_dim=3_7)




     def 		snake_case_		(				self):
         self.config_tester.run_common_tests()




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_model(*a__)




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_lm_head(*a__)




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_simple_qa(*a__)




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_qa(*a__)




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_sequence_classif(*a__)




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_token_classif(*a__)




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_for_multiple_choice(*a__)




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__=False ,						a__=1):
         self.assertIsInstance(a__ ,						a__)
         self.assertListEqual(
             [isinstance(a__ ,						a__) for iter_attentions in attentions] ,						[True] * len(a__))
         self.assertEqual(len(a__) ,						(max_length - min_length) * num_beam_groups)

         for idx, iter_attentions in enumerate(a__):
             # adds PAD dummy token
             A__		 =					min_length + idx + 1
             A__		 =					min_length + idx + 1

             A__		 =					(
                 batch_size * num_beam_groups,
                 config.num_attention_heads,
                 tgt_len,
                 src_len,
             )
             # check attn size
             self.assertListEqual(
                 [layer_attention.shape for layer_attention in iter_attentions] ,						[expected_shape] * len(a__))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__=False ,						a__=1):
         self.assertIsInstance(a__ ,						a__)
         self.assertListEqual(
             [isinstance(a__ ,						a__) for iter_hidden_states in hidden_states] ,						[True] * len(a__) ,						)
         self.assertEqual(len(a__) ,						(max_length - min_length) * num_beam_groups)

         for idx, iter_hidden_states in enumerate(a__):
             # adds PAD dummy token
             A__		 =					min_length + idx + 1
             A__		 =					(batch_size * num_beam_groups, seq_len, config.hidden_size)
             # check hidden size
             self.assertListEqual(
                 [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states] ,						[expected_shape] * len(a__) ,						)
         pass




     @slow
     def 		snake_case_		(				self):
         for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             A__		 =					XLMModel.from_pretrained(a__)
             self.assertIsNotNone(a__)







@require_torch
class     _UpperCAmelCase			(       unittest.TestCase     ):




     @slow
     def 		snake_case_		(				self):
         A__		 =					XLMWithLMHeadModel.from_pretrained('''xlm-mlm-en-2048''')
         model.to(a__)
         A__		 =					torch.tensor([[1_4, 4_4_7]] ,						dtype=torch.long ,						device=a__)  # the president
         A__		 =					[
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
         ]  # the president the president the president the president the president the president the president the president the president the president
         # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
         A__		 =					model.generate(a__ ,						do_sample=a__)
         self.assertListEqual(output_ids[0].cpu().numpy().tolist() ,						a__)





import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  str:
    A__		 =					1_0
    A__		 =					datasets.Features(
        {
            '''tokens''': datasets.Sequence(datasets.Value('''string'''	)	),
            '''labels''': datasets.Sequence(datasets.ClassLabel(names=['''negative''', '''positive''']	)	),
            '''answers''': datasets.Sequence(
                {
                    '''text''': datasets.Value('''string'''	),
                    '''answer_start''': datasets.Value('''int32'''	),
                }	),
            '''id''': datasets.Value('''int64'''	),
        }	)
    A__		 =					datasets.Dataset.from_dict(
        {
            '''tokens''': [['''foo'''] * 5] * n,
            '''labels''': [[1] * 5] * n,
            '''answers''': [{'''answer_start''': [9_7], '''text''': ['''1976''']}] * 1_0,
            '''id''': list(range(UpperCamelCase_	)	),
        } ,       features=UpperCamelCase_ ,       )
    return dataset



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       int	)->  Optional[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''file.arrow'''	)
    dataset.map(cache_file_name=UpperCamelCase_	)
    return filename


# FILE_CONTENT + files


_lowercase						  =		"\\n    Text data.\n    Second line of data."



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  List[Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt'''
    A__		 =					FILE_CONTENT
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_	)
    return filename



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  Optional[Any]:
    import bza

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.bz2'''
    A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
    with bza.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any]	)->  int:
    import gzip

    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''file.txt.gz'''	)
    A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
    with gzip.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  Any:
    if datasets.config.LZ4_AVAILABLE:
        import lza.frame

        A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.lz4'''
        A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
        with lza.frame.open(UpperCamelCase_ ,       '''wb'''	) as f:
            f.write(UpperCamelCase_	)
        return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[Any] ,       UpperCamelCase_				:       Tuple	)->  int:
    if datasets.config.PY7ZR_AVAILABLE:
        import pyazr

        A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.7z'''
        with pyazr.SevenZipFile(UpperCamelCase_ ,       '''w'''	) as archive:
            archive.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Tuple ,       UpperCamelCase_				:       int	)->  Optional[Any]:
    import tarfile

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.tar'''
    with tarfile.TarFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.add(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  str:
    import lzma

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.xz'''
    A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
    with lzma.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int] ,       UpperCamelCase_				:       List[str]	)->  List[str]:
    import zipfile

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Tuple	)->  str:
    if datasets.config.ZSTANDARD_AVAILABLE:
        import zstandard as zstd

        A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.zst'''
        A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
        with zstd.open(UpperCamelCase_ ,       '''wb'''	) as f:
            f.write(UpperCamelCase_	)
        return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  int:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.xml'''
    A__		 =					textwrap.dedent(
        '''\
    <?xml version="1.0" encoding="UTF-8" ?>
    <tmx version="1.4">
      <header segtype="sentence" srclang="ca" />
      <body>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang="en"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang="en"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang="en"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang="en"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang="en"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_	)
    return filename


_lowercase						  =		[
    {"col_1": "0", "col_2": 0, "col_3": 0.0},
    {"col_1": "1", "col_2": 1, "col_3": 1.0},
    {"col_1": "2", "col_2": 2, "col_3": 2.0},
    {"col_1": "3", "col_2": 3, "col_3": 3.0},
]
_lowercase						  =		[
    {"col_1": "4", "col_2": 4, "col_3": 4.0},
    {"col_1": "5", "col_2": 5, "col_3": 5.0},
]
_lowercase						  =		{
    "col_1": ["0", "1", "2", "3"],
    "col_2": [0, 1, 2, 3],
    "col_3": [0.0, 1.0, 2.0, 3.0],
}

_lowercase						  =		[
    {"col_3": 0.0, "col_1": "0", "col_2": 0},
    {"col_3": 1.0, "col_1": "1", "col_2": 1},
]

_lowercase						  =		[
    {"col_1": "s0", "col_2": 0, "col_3": 0.0},
    {"col_1": "s1", "col_2": 1, "col_3": 1.0},
    {"col_1": "s2", "col_2": 2, "col_3": 2.0},
    {"col_1": "s3", "col_2": 3, "col_3": 3.0},
]



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  str:
    return DATA_DICT_OF_LISTS



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[Any]	)->  List[str]:
    A__		 =					datasets.Dataset.from_dict(UpperCamelCase_	)
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.arrow'''	)
    dataset.map(cache_file_name=UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  List[str]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.sqlite'''	)
    with contextlib.closing(sqlitea.connect(UpperCamelCase_	)	) as con:
        A__		 =					con.cursor()
        cur.execute('''CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'''	)
        for item in DATA:
            cur.execute('''INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)''' ,       tuple(item.values()	)	)
        con.commit()
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any]	)->  Tuple:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.csv'''	)
    with open(UpperCamelCase_ ,       '''w''' ,       newline=''''''	) as f:
        A__		 =					csv.DictWriter(UpperCamelCase_ ,       fieldnames=['''col_1''', '''col_2''', '''col_3''']	)
        writer.writeheader()
        for item in DATA:
            writer.writerow(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  List[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset2.csv'''	)
    with open(UpperCamelCase_ ,       '''w''' ,       newline=''''''	) as f:
        A__		 =					csv.DictWriter(UpperCamelCase_ ,       fieldnames=['''col_1''', '''col_2''', '''col_3''']	)
        writer.writeheader()
        for item in DATA:
            writer.writerow(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       Optional[int]	)->  List[str]:
    import bza

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.csv.bz2'''
    with open(UpperCamelCase_ ,       '''rb'''	) as f:
        A__		 =					f.read()
    # data = bytes(FILE_CONTENT, "utf-8")
    with bza.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str ,       UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       Optional[int]	)->  str:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.csv.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       str ,       UpperCamelCase_				:       Any	)->  List[Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.csv.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(csv_path.replace('''.csv''' ,       '''.CSV'''	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(csva_path.replace('''.csv''' ,       '''.CSV'''	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str ,       UpperCamelCase_				:       str ,       UpperCamelCase_				:       Optional[Any]	)->  Tuple:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_with_dir.csv.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  Optional[int]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.parquet'''	)
    A__		 =					pa.schema(
        {
            '''col_1''': pa.string(),
            '''col_2''': pa.intaa(),
            '''col_3''': pa.floataa(),
        }	)
    with open(UpperCamelCase_ ,       '''wb'''	) as f:
        A__		 =					pq.ParquetWriter(UpperCamelCase_ ,       schema=UpperCamelCase_	)
        A__		 =					pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(UpperCamelCase_	)	)] for k in DATA[0]} ,       schema=UpperCamelCase_	)
        writer.write_table(UpperCamelCase_	)
        writer.close()
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  str:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.json'''	)
    A__		 =					{'''data''': DATA}
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        json.dump(UpperCamelCase_ ,       UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  Optional[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.json'''	)
    A__		 =					{'''data''': DATA_DICT_OF_LISTS}
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        json.dump(UpperCamelCase_ ,       UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  List[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  Tuple:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset2.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int	)->  Optional[int]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset_312.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA_312:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int	)->  Union[str, Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset-str.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA_STR:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any] ,       UpperCamelCase_				:       str	)->  List[str]:
    import gzip

    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.txt.gz'''	)
    with open(UpperCamelCase_ ,       '''rb'''	) as orig_file:
        with gzip.open(UpperCamelCase_ ,       '''wb'''	) as zipped_file:
            zipped_file.writelines(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Any ,       UpperCamelCase_				:       Optional[Any]	)->  List[Any]:
    import gzip

    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl.gz'''	)
    with open(UpperCamelCase_ ,       '''rb'''	) as orig_file:
        with gzip.open(UpperCamelCase_ ,       '''wb'''	) as zipped_file:
            zipped_file.writelines(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       List[str]	)->  Optional[int]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       Optional[int]	)->  Union[str, Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_nested.jsonl.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''nested''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       Any	)->  Any:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_with_dir.jsonl.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict ,       UpperCamelCase_				:       Any ,       UpperCamelCase_				:       str	)->  Tuple:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl.tar'''
    with tarfile.TarFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.add(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.add(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int] ,       UpperCamelCase_				:       int ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       List[str]	)->  str:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_nested.jsonl.tar'''
    with tarfile.TarFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.add(UpperCamelCase_ ,       arcname=os.path.join('''nested''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Any	)->  List[str]:
    A__		 =					['''0''', '''1''', '''2''', '''3''']
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.txt'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in data:
            f.write(item + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int	)->  Dict:
    A__		 =					['''0''', '''1''', '''2''', '''3''']
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset2.txt'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in data:
            f.write(item + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any]	)->  Union[str, Any]:
    A__		 =					['''0''', '''1''', '''2''', '''3''']
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.abc'''
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in data:
            f.write(item + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       Any ,       UpperCamelCase_				:       Optional[Any]	)->  List[str]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.text.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Any ,       UpperCamelCase_				:       List[Any] ,       UpperCamelCase_				:       Optional[int]	)->  Tuple:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_with_dir.text.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       Tuple ,       UpperCamelCase_				:       Tuple	)->  Any:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.ext.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename('''unsupported.ext'''	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename('''unsupported_2.ext'''	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  Any:
    A__		 =					'''\n'''.join(['''First''', '''Second\u2029with Unicode new line''', '''Third''']	)
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset_with_unicode_new_lines.txt'''	)
    with open(UpperCamelCase_ ,       '''w''' ,       encoding='''utf-8'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  Tuple:
    return os.path.join('''tests''' ,       '''features''' ,       '''data''' ,       '''test_image_rgb.jpg'''	)



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  Dict:
    return os.path.join('''tests''' ,       '''features''' ,       '''data''' ,       '''test_audio_44100.wav'''	)



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any] ,       UpperCamelCase_				:       Union[str, Any]	)->  List[Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.img.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	).replace('''.jpg''' ,       '''2.jpg'''	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any]	)->  str:
    A__		 =					tmp_path_factory.mktemp('''data_dir'''	)

    (data_dir / "subdir").mkdir()
    with open(data_dir / '''subdir''' / '''train.txt''' ,       '''w'''	) as f:
        f.write('''foo\n''' * 1_0	)
    with open(data_dir / '''subdir''' / '''test.txt''' ,       '''w'''	) as f:
        f.write('''bar\n''' * 1_0	)
    # hidden file
    with open(data_dir / '''subdir''' / '''.test.txt''' ,       '''w'''	) as f:
        f.write('''bar\n''' * 1_0	)

    # hidden directory
    (data_dir / ".subdir").mkdir()
    with open(data_dir / '''.subdir''' / '''train.txt''' ,       '''w'''	) as f:
        f.write('''foo\n''' * 1_0	)
    with open(data_dir / '''.subdir''' / '''test.txt''' ,       '''w'''	) as f:
        f.write('''bar\n''' * 1_0	)

    return data_dir
