
'''simple docstring'''





from __future__ import annotations

import unittest

from transformers import XGLMConfig, XGLMTokenizer, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers.models.xglm.modeling_tf_xglm import (
						    TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFXGLMForCausalLM,
						    TFXGLMModel,
						)

@require_tf
class 						a   :
				_snake_case		    = XGLMConfig
				_snake_case		    = {}
				_snake_case		    = '''gelu'''




				def __init__(							self	:				Optional[int],    SCREAMING_SNAKE_CASE_	:				Optional[int],    SCREAMING_SNAKE_CASE_	:				List[Any]=14,    SCREAMING_SNAKE_CASE_	:				List[str]=7,    SCREAMING_SNAKE_CASE_	:				List[Any]=True,    SCREAMING_SNAKE_CASE_	:				str=True,    SCREAMING_SNAKE_CASE_	:				List[str]=True,    SCREAMING_SNAKE_CASE_	:				int=99,    SCREAMING_SNAKE_CASE_	:				int=32,    SCREAMING_SNAKE_CASE_	:				Union[str, Any]=2,    SCREAMING_SNAKE_CASE_	:				Dict=4,    SCREAMING_SNAKE_CASE_	:				List[str]=37,    SCREAMING_SNAKE_CASE_	:				Union[str, Any]="gelu",    SCREAMING_SNAKE_CASE_	:				Union[str, Any]=0.1,    SCREAMING_SNAKE_CASE_	:				Union[str, Any]=0.1,    SCREAMING_SNAKE_CASE_	:				Optional[Any]=5_12,    SCREAMING_SNAKE_CASE_	:				Optional[int]=0.02,    ):
										snake_case       :     Optional[int] =    parent
										snake_case       :     Tuple =    batch_size
										snake_case       :     Dict =    seq_length
										snake_case       :     int =    is_training
										snake_case       :     Union[str, Any] =    use_input_mask
										snake_case       :     Any =    use_labels
										snake_case       :     Any =    vocab_size
										snake_case       :     str =    d_model
										snake_case       :     Any =    num_hidden_layers
										snake_case       :     Any =    num_attention_heads
										snake_case       :     Optional[int] =    ffn_dim
										snake_case       :     str =    activation_function
										snake_case       :     Optional[int] =    activation_dropout
										snake_case       :     Optional[int] =    attention_dropout
										snake_case       :     Dict =    max_position_embeddings
										snake_case       :     Any =    initializer_range
										snake_case       :     Optional[Any] =    None
										snake_case       :     Dict =    0
										snake_case       :     List[Any] =    2
										snake_case       :     List[Any] =    1




				def 	__snake_case							(							self	:				int      ):
										return XGLMConfig.from_pretrained('''facebook/xglm-564M'''      )




				def 	__snake_case							(							self	:				Optional[Any]      ):
										snake_case       :     int =    tf.clip_by_value(
										    ids_tensor([self.batch_size, self.seq_length],    self.vocab_size      ),    clip_value_min=0,    clip_value_max=3      )

										snake_case       :     Union[str, Any] =    None
										if self.use_input_mask:
																snake_case       :     Tuple =    random_attention_mask([self.batch_size, self.seq_length]      )

										snake_case       :     List[str] =    self.get_config()

										snake_case       :     List[str] =    floats_tensor([self.num_hidden_layers, self.num_attention_heads],    2      )

										return (
										    config,
										    input_ids,
										    input_mask,
										    head_mask,
										)




				def 	__snake_case							(							self	:				str      ):
										return XGLMConfig(
										    vocab_size=self.vocab_size,    d_model=self.hidden_size,    num_layers=self.num_hidden_layers,    attention_heads=self.num_attention_heads,    ffn_dim=self.ffn_dim,    activation_function=self.activation_function,    activation_dropout=self.activation_dropout,    attention_dropout=self.attention_dropout,    max_position_embeddings=self.max_position_embeddings,    initializer_range=self.initializer_range,    use_cache=SCREAMING_SNAKE_CASE_,    bos_token_id=self.bos_token_id,    eos_token_id=self.eos_token_id,    pad_token_id=self.pad_token_id,    return_dict=SCREAMING_SNAKE_CASE_,    )




				def 	__snake_case							(							self	:				Any      ):
										snake_case       :     Optional[int] =    self.prepare_config_and_inputs()

										(
										    (
										    snake_case
										),					(
										    snake_case
										),					(
										    snake_case
										),					(
										    snake_case
										),					
										)       :     int =    config_and_inputs

										snake_case       :     Union[str, Any] =    {
										    '''input_ids''': input_ids,
										    '''head_mask''': head_mask,
										}

										return config, inputs_dict




@require_tf
class 						a   (				__magic_name__				,__magic_name__				,unittest.TestCase    ):
				_snake_case		    = (TFXGLMModel, TFXGLMForCausalLM) if is_tf_available() else ()
				_snake_case		    = (TFXGLMForCausalLM,) if is_tf_available() else ()
				_snake_case		    = (
				    {'''feature-extraction''': TFXGLMModel, '''text-generation''': TFXGLMForCausalLM} if is_tf_available() else {}
				)
				_snake_case		    = False
				_snake_case		    = False
				_snake_case		    = False




				def 	__snake_case							(							self	:				Optional[int]      ):
										snake_case       :     Tuple =    TFXGLMModelTester(self      )
										snake_case       :     Any =    ConfigTester(self,    config_class=SCREAMING_SNAKE_CASE_,    n_embd=37      )




				def 	__snake_case							(							self	:				Union[str, Any]      ):
										self.config_tester.run_common_tests()




				@slow
				def 	__snake_case							(							self	:				Optional[Any]      ):
										for model_name in TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																snake_case       :     str =    TFXGLMModel.from_pretrained(SCREAMING_SNAKE_CASE_      )
																self.assertIsNotNone(SCREAMING_SNAKE_CASE_      )




				@unittest.skip(reason='''Currently, model embeddings are going to undergo a major refactor.'''      )
				def 	__snake_case							(							self	:				int      ):
										super().test_resize_token_embeddings()




@require_tf
class 						a   (				unittest.TestCase    ):




				@slow
				def 	__snake_case							(							self	:				List[Any],    SCREAMING_SNAKE_CASE_	:				List[Any]=True      ):
										snake_case       :     List[str] =    TFXGLMForCausalLM.from_pretrained('''facebook/xglm-564M'''      )
										snake_case       :     Optional[Any] =    tf.convert_to_tensor([[2, 2_68, 98_65]],    dtype=tf.intaa      )  # The dog
										# </s> The dog is a very friendly dog. He is very affectionate and loves to play with other
										# fmt: off
										snake_case       :     Optional[int] =    [2, 2_68, 98_65, 67, 11, 19_88, 5_72_52, 98_65, 5, 9_84, 67, 19_88, 21_38_38, 16_58, 53, 7_04_46, 33, 66_57, 2_78, 15_81]
										# fmt: on
										snake_case       :     List[Any] =    model.generate(SCREAMING_SNAKE_CASE_,    do_sample=SCREAMING_SNAKE_CASE_,    num_beams=1      )
										if verify_outputs:
																self.assertListEqual(output_ids[0].numpy().tolist(),    SCREAMING_SNAKE_CASE_      )




				@slow
				def 	__snake_case							(							self	:				str      ):
										snake_case       :     Optional[Any] =    XGLMTokenizer.from_pretrained('''facebook/xglm-564M'''      )
										snake_case       :     Any =    TFXGLMForCausalLM.from_pretrained('''facebook/xglm-564M'''      )

										tf.random.set_seed(0      )
										snake_case       :     Tuple =    tokenizer('''Today is a nice day and''',    return_tensors='''tf'''      )
										snake_case       :     Optional[int] =    tokenized.input_ids
										# forces the generation to happen on CPU, to avoid GPU-related quirks (and assure same output regardless of the available devices)
										with tf.device(''':/CPU:0'''      ):
																snake_case       :     Dict =    model.generate(SCREAMING_SNAKE_CASE_,    do_sample=SCREAMING_SNAKE_CASE_,    seed=[7, 0]      )
										snake_case       :     Union[str, Any] =    tokenizer.decode(output_ids[0],    skip_special_tokens=SCREAMING_SNAKE_CASE_      )

										snake_case       :     Optional[Any] =    (
										    '''Today is a nice day and warm evening here over Southern Alberta!! Today when they closed schools due'''
										)
										self.assertEqual(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_      )




				@slow
				def 	__snake_case							(							self	:				Dict      ):
										snake_case       :     Union[str, Any] =    TFXGLMForCausalLM.from_pretrained('''facebook/xglm-564M'''      )
										snake_case       :     str =    XGLMTokenizer.from_pretrained('''facebook/xglm-564M'''      )

										snake_case       :     Union[str, Any] =    '''left'''

										# use different length sentences to test batching
										snake_case       :     List[Any] =    [
										    '''This is an extremelly long sentence that only exists to test the ability of the model to cope with '''
										    '''left-padding, such as in batched generation. The output for the sequence below should be the same '''
										    '''regardless of whether left padding is applied or not. When''',
										    '''Hello, my dog is a little''',
										]

										snake_case       :     Optional[Any] =    tokenizer(SCREAMING_SNAKE_CASE_,    return_tensors='''tf''',    padding=SCREAMING_SNAKE_CASE_      )
										snake_case       :     List[Any] =    inputs['''input_ids''']

										snake_case       :     Dict =    model.generate(input_ids=SCREAMING_SNAKE_CASE_,    attention_mask=inputs['''attention_mask'''],    max_new_tokens=12      )

										snake_case       :     Any =    tokenizer(sentences[0],    return_tensors='''tf'''      ).input_ids
										snake_case       :     Optional[int] =    model.generate(input_ids=SCREAMING_SNAKE_CASE_,    max_new_tokens=12      )

										snake_case       :     List[Any] =    tokenizer(sentences[1],    return_tensors='''tf'''      ).input_ids
										snake_case       :     Dict =    model.generate(input_ids=SCREAMING_SNAKE_CASE_,    max_new_tokens=12      )

										snake_case       :     Any =    tokenizer.batch_decode(SCREAMING_SNAKE_CASE_,    skip_special_tokens=SCREAMING_SNAKE_CASE_      )
										snake_case       :     Dict =    tokenizer.decode(output_non_padded[0],    skip_special_tokens=SCREAMING_SNAKE_CASE_      )
										snake_case       :     Dict =    tokenizer.decode(output_padded[0],    skip_special_tokens=SCREAMING_SNAKE_CASE_      )

										snake_case       :     Optional[Any] =    [
										    '''This is an extremelly long sentence that only exists to test the ability of the model to cope with '''
										    '''left-padding, such as in batched generation. The output for the sequence below should be the same '''
										    '''regardless of whether left padding is applied or not. When left padding is applied, the sequence will be '''
										    '''a single''',
										    '''Hello, my dog is a little bit of a shy one, but he is very friendly''',
										]
										self.assertListEqual(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_      )
										self.assertListEqual(SCREAMING_SNAKE_CASE_,    [non_padded_sentence, padded_sentence]      )



'''simple docstring'''





import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config

@pytest.fixture(scope='''session'''       )
def      A (       ):
						snake_case       :     Dict =    10
						snake_case       :     List[Any] =    datasets.Features(
						    {
						        '''tokens''': datasets.Sequence(datasets.Value('''string'''       )       ),
						        '''labels''': datasets.Sequence(datasets.ClassLabel(names=['''negative''', '''positive''']       )       ),
						        '''answers''': datasets.Sequence(
						            {
						                '''text''': datasets.Value('''string'''       ),
						                '''answer_start''': datasets.Value('''int32'''       ),
						            }       ),
						        '''id''': datasets.Value('''int64'''       ),
						    }       )
						snake_case       :     Dict =    datasets.Dataset.from_dict(
						    {
						        '''tokens''': [['''foo'''] * 5] * n,
						        '''labels''': [[1] * 5] * n,
						        '''answers''': [{'''answer_start''': [97], '''text''': ['''1976''']}] * 10,
						        '''id''': list(range(A_       )       ),
						    }    ,							features=A_    ,							)
						return dataset

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[Any]    ,							A_							:					Optional[Any]       ):
						snake_case       :     Any =    str(tmp_path_factory.mktemp('''data'''       ) / '''file.arrow'''       )
						dataset.map(cache_file_name=A_       )
						return filename


# FILE_CONTENT + files


UpperCAmelCase    					=   "\\n    Text data.\n    Second line of data."

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[str]       ):
						snake_case       :     Any =    tmp_path_factory.mktemp('''data'''       ) / '''file.txt'''
						snake_case       :     Union[str, Any] =    FILE_CONTENT
						with open(A_    ,							'''w'''       ) as f:
												f.write(A_       )
						return filename

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Optional[Any]       ):
						import bza

						snake_case       :     Tuple =    tmp_path_factory.mktemp('''data'''       ) / '''file.txt.bz2'''
						snake_case       :     Union[str, Any] =    bytes(A_    ,							'''utf-8'''       )
						with bza.open(A_    ,							'''wb'''       ) as f:
												f.write(A_       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[Any]       ):
						import gzip

						snake_case       :     Any =    str(tmp_path_factory.mktemp('''data'''       ) / '''file.txt.gz'''       )
						snake_case       :     Any =    bytes(A_    ,							'''utf-8'''       )
						with gzip.open(A_    ,							'''wb'''       ) as f:
												f.write(A_       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Optional[int]       ):
						if datasets.config.LZ4_AVAILABLE:
												import lza.frame

												snake_case       :     Union[str, Any] =    tmp_path_factory.mktemp('''data'''       ) / '''file.txt.lz4'''
												snake_case       :     Tuple =    bytes(A_    ,							'''utf-8'''       )
												with lza.frame.open(A_    ,							'''wb'''       ) as f:
																		f.write(A_       )
												return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Union[str, Any]    ,							A_							:					str       ):
						if datasets.config.PY7ZR_AVAILABLE:
												import pyazr

												snake_case       :     Optional[Any] =    tmp_path_factory.mktemp('''data'''       ) / '''file.txt.7z'''
												with pyazr.SevenZipFile(A_    ,							'''w'''       ) as archive:
																		archive.write(A_    ,							arcname=os.path.basename(A_       )       )
												return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[str]    ,							A_							:					Optional[int]       ):
						import tarfile

						snake_case       :     Union[str, Any] =    tmp_path_factory.mktemp('''data'''       ) / '''file.txt.tar'''
						with tarfile.TarFile(A_    ,							'''w'''       ) as f:
												f.add(A_    ,							arcname=os.path.basename(A_       )       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Any       ):
						import lzma

						snake_case       :     Any =    tmp_path_factory.mktemp('''data'''       ) / '''file.txt.xz'''
						snake_case       :     Union[str, Any] =    bytes(A_    ,							'''utf-8'''       )
						with lzma.open(A_    ,							'''wb'''       ) as f:
												f.write(A_       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Optional[int]    ,							A_							:					Union[str, Any]       ):
						import zipfile

						snake_case       :     List[str] =    tmp_path_factory.mktemp('''data'''       ) / '''file.txt.zip'''
						with zipfile.ZipFile(A_    ,							'''w'''       ) as f:
												f.write(A_    ,							arcname=os.path.basename(A_       )       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[str]       ):
						if datasets.config.ZSTANDARD_AVAILABLE:
												import zstandard as zstd

												snake_case       :     Tuple =    tmp_path_factory.mktemp('''data'''       ) / '''file.txt.zst'''
												snake_case       :     List[str] =    bytes(A_    ,							'''utf-8'''       )
												with zstd.open(A_    ,							'''wb'''       ) as f:
																		f.write(A_       )
												return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					str       ):
						snake_case       :     Optional[int] =    tmp_path_factory.mktemp('''data'''       ) / '''file.xml'''
						snake_case       :     Any =    textwrap.dedent(
						    '''\
    <?xml version="1.0" encoding="UTF-8" ?>
    <tmx version="1.4">
      <header segtype="sentence" srclang="ca" />
      <body>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang="en"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang="en"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang="en"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang="en"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang="en"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>'''       )
						with open(A_    ,							'''w'''       ) as f:
												f.write(A_       )
						return filename


UpperCAmelCase    					=   [
    {"col_1": "0", "col_2": 0, "col_3": 0.0},
    {"col_1": "1", "col_2": 1, "col_3": 1.0},
    {"col_1": "2", "col_2": 2, "col_3": 2.0},
    {"col_1": "3", "col_2": 3, "col_3": 3.0},
]
UpperCAmelCase    					=   [
    {"col_1": "4", "col_2": 4, "col_3": 4.0},
    {"col_1": "5", "col_2": 5, "col_3": 5.0},
]
UpperCAmelCase    					=   {
    "col_1": ["0", "1", "2", "3"],
    "col_2": [0, 1, 2, 3],
    "col_3": [0.0, 1.0, 2.0, 3.0],
}

UpperCAmelCase    					=   [
    {"col_3": 0.0, "col_1": "0", "col_2": 0},
    {"col_3": 1.0, "col_1": "1", "col_2": 1},
]

UpperCAmelCase    					=   [
    {"col_1": "s0", "col_2": 0, "col_3": 0.0},
    {"col_1": "s1", "col_2": 1, "col_3": 1.0},
    {"col_1": "s2", "col_2": 2, "col_3": 2.0},
    {"col_1": "s3", "col_2": 3, "col_3": 3.0},
]

@pytest.fixture(scope='''session'''       )
def      A (       ):
						return DATA_DICT_OF_LISTS

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					str       ):
						snake_case       :     str =    datasets.Dataset.from_dict(A_       )
						snake_case       :     List[Any] =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset.arrow'''       )
						dataset.map(cache_file_name=A_       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					str       ):
						snake_case       :     Optional[int] =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset.sqlite'''       )
						with contextlib.closing(sqlitea.connect(A_       )       ) as con:
												snake_case       :     Optional[Any] =    con.cursor()
												cur.execute('''CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'''       )
												for item in DATA:
																		cur.execute('''INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'''    ,							tuple(item.values()       )       )
												con.commit()
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[str]       ):
						snake_case       :     Any =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset.csv'''       )
						with open(A_    ,							'''w'''    ,							newline=''''''       ) as f:
												snake_case       :     Optional[int] =    csv.DictWriter(A_    ,							fieldnames=['''col_1''', '''col_2''', '''col_3''']       )
												writer.writeheader()
												for item in DATA:
																		writer.writerow(A_       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Optional[int]       ):
						snake_case       :     Optional[int] =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset2.csv'''       )
						with open(A_    ,							'''w'''    ,							newline=''''''       ) as f:
												snake_case       :     int =    csv.DictWriter(A_    ,							fieldnames=['''col_1''', '''col_2''', '''col_3''']       )
												writer.writeheader()
												for item in DATA:
																		writer.writerow(A_       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Optional[Any]    ,							A_							:					Optional[int]       ):
						import bza

						snake_case       :     Optional[int] =    tmp_path_factory.mktemp('''data'''       ) / '''dataset.csv.bz2'''
						with open(A_    ,							'''rb'''       ) as f:
												snake_case       :     Any =    f.read()
						# data = bytes(FILE_CONTENT, "utf-8")
						with bza.open(A_    ,							'''wb'''       ) as f:
												f.write(A_       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					str    ,							A_							:					int    ,							A_							:					List[Any]       ):
						snake_case       :     Optional[Any] =    tmp_path_factory.mktemp('''data'''       ) / '''dataset.csv.zip'''
						with zipfile.ZipFile(A_    ,							'''w'''       ) as f:
												f.write(A_    ,							arcname=os.path.basename(A_       )       )
												f.write(A_    ,							arcname=os.path.basename(A_       )       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Dict    ,							A_							:					Optional[Any]    ,							A_							:					Dict       ):
						snake_case       :     str =    tmp_path_factory.mktemp('''data'''       ) / '''dataset.csv.zip'''
						with zipfile.ZipFile(A_    ,							'''w'''       ) as f:
												f.write(A_    ,							arcname=os.path.basename(csv_path.replace('''.csv'''    ,							'''.CSV'''       )       )       )
												f.write(A_    ,							arcname=os.path.basename(csva_path.replace('''.csv'''    ,							'''.CSV'''       )       )       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Dict    ,							A_							:					Dict    ,							A_							:					Optional[int]       ):
						snake_case       :     Dict =    tmp_path_factory.mktemp('''data'''       ) / '''dataset_with_dir.csv.zip'''
						with zipfile.ZipFile(A_    ,							'''w'''       ) as f:
												f.write(A_    ,							arcname=os.path.join('''main_dir'''    ,							os.path.basename(A_       )       )       )
												f.write(A_    ,							arcname=os.path.join('''main_dir'''    ,							os.path.basename(A_       )       )       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Dict       ):
						snake_case       :     List[str] =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset.parquet'''       )
						snake_case       :     Tuple =    pa.schema(
						    {
						        '''col_1''': pa.string(),
						        '''col_2''': pa.intaa(),
						        '''col_3''': pa.floataa(),
						    }       )
						with open(A_    ,							'''wb'''       ) as f:
												snake_case       :     Dict =    pq.ParquetWriter(A_    ,							schema=A_       )
												snake_case       :     str =    pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(A_       )       )] for k in DATA[0]}    ,							schema=A_       )
												writer.write_table(A_       )
												writer.close()
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[str]       ):
						snake_case       :     Any =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset.json'''       )
						snake_case       :     Optional[int] =    {'''data''': DATA}
						with open(A_    ,							'''w'''       ) as f:
												json.dump(A_    ,							A_       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					str       ):
						snake_case       :     List[str] =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset.json'''       )
						snake_case       :     Any =    {'''data''': DATA_DICT_OF_LISTS}
						with open(A_    ,							'''w'''       ) as f:
												json.dump(A_    ,							A_       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Optional[Any]       ):
						snake_case       :     Optional[Any] =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset.jsonl'''       )
						with open(A_    ,							'''w'''       ) as f:
												for item in DATA:
																		f.write(json.dumps(A_       ) + '''\n'''       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[Any]       ):
						snake_case       :     Optional[int] =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset2.jsonl'''       )
						with open(A_    ,							'''w'''       ) as f:
												for item in DATA:
																		f.write(json.dumps(A_       ) + '''\n'''       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[Any]       ):
						snake_case       :     Dict =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset_312.jsonl'''       )
						with open(A_    ,							'''w'''       ) as f:
												for item in DATA_312:
																		f.write(json.dumps(A_       ) + '''\n'''       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					int       ):
						snake_case       :     Dict =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset-str.jsonl'''       )
						with open(A_    ,							'''w'''       ) as f:
												for item in DATA_STR:
																		f.write(json.dumps(A_       ) + '''\n'''       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Optional[int]    ,							A_							:					Optional[int]       ):
						import gzip

						snake_case       :     Dict =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset.txt.gz'''       )
						with open(A_    ,							'''rb'''       ) as orig_file:
												with gzip.open(A_    ,							'''wb'''       ) as zipped_file:
																		zipped_file.writelines(A_       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Optional[int]    ,							A_							:					Any       ):
						import gzip

						snake_case       :     List[Any] =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset.jsonl.gz'''       )
						with open(A_    ,							'''rb'''       ) as orig_file:
												with gzip.open(A_    ,							'''wb'''       ) as zipped_file:
																		zipped_file.writelines(A_       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					str    ,							A_							:					List[str]    ,							A_							:					Optional[int]       ):
						snake_case       :     int =    tmp_path_factory.mktemp('''data'''       ) / '''dataset.jsonl.zip'''
						with zipfile.ZipFile(A_    ,							'''w'''       ) as f:
												f.write(A_    ,							arcname=os.path.basename(A_       )       )
												f.write(A_    ,							arcname=os.path.basename(A_       )       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[str]    ,							A_							:					Any    ,							A_							:					Union[str, Any]    ,							A_							:					int       ):
						snake_case       :     List[str] =    tmp_path_factory.mktemp('''data'''       ) / '''dataset_nested.jsonl.zip'''
						with zipfile.ZipFile(A_    ,							'''w'''       ) as f:
												f.write(A_    ,							arcname=os.path.join('''nested'''    ,							os.path.basename(A_       )       )       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[str]    ,							A_							:					Dict    ,							A_							:					Any       ):
						snake_case       :     Optional[Any] =    tmp_path_factory.mktemp('''data'''       ) / '''dataset_with_dir.jsonl.zip'''
						with zipfile.ZipFile(A_    ,							'''w'''       ) as f:
												f.write(A_    ,							arcname=os.path.join('''main_dir'''    ,							os.path.basename(A_       )       )       )
												f.write(A_    ,							arcname=os.path.join('''main_dir'''    ,							os.path.basename(A_       )       )       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[Any]    ,							A_							:					Tuple    ,							A_							:					List[str]       ):
						snake_case       :     int =    tmp_path_factory.mktemp('''data'''       ) / '''dataset.jsonl.tar'''
						with tarfile.TarFile(A_    ,							'''w'''       ) as f:
												f.add(A_    ,							arcname=os.path.basename(A_       )       )
												f.add(A_    ,							arcname=os.path.basename(A_       )       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Tuple    ,							A_							:					Any    ,							A_							:					Tuple    ,							A_							:					Optional[Any]       ):
						snake_case       :     List[Any] =    tmp_path_factory.mktemp('''data'''       ) / '''dataset_nested.jsonl.tar'''
						with tarfile.TarFile(A_    ,							'''w'''       ) as f:
												f.add(A_    ,							arcname=os.path.join('''nested'''    ,							os.path.basename(A_       )       )       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					int       ):
						snake_case       :     int =    ['''0''', '''1''', '''2''', '''3''']
						snake_case       :     Optional[int] =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset.txt'''       )
						with open(A_    ,							'''w'''       ) as f:
												for item in data:
																		f.write(item + '''\n'''       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[str]       ):
						snake_case       :     Union[str, Any] =    ['''0''', '''1''', '''2''', '''3''']
						snake_case       :     Optional[int] =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset2.txt'''       )
						with open(A_    ,							'''w'''       ) as f:
												for item in data:
																		f.write(item + '''\n'''       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Optional[int]       ):
						snake_case       :     Optional[int] =    ['''0''', '''1''', '''2''', '''3''']
						snake_case       :     str =    tmp_path_factory.mktemp('''data'''       ) / '''dataset.abc'''
						with open(A_    ,							'''w'''       ) as f:
												for item in data:
																		f.write(item + '''\n'''       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Optional[Any]    ,							A_							:					Any    ,							A_							:					Union[str, Any]       ):
						snake_case       :     Dict =    tmp_path_factory.mktemp('''data'''       ) / '''dataset.text.zip'''
						with zipfile.ZipFile(A_    ,							'''w'''       ) as f:
												f.write(A_    ,							arcname=os.path.basename(A_       )       )
												f.write(A_    ,							arcname=os.path.basename(A_       )       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					List[str]    ,							A_							:					Union[str, Any]    ,							A_							:					Optional[Any]       ):
						snake_case       :     List[Any] =    tmp_path_factory.mktemp('''data'''       ) / '''dataset_with_dir.text.zip'''
						with zipfile.ZipFile(A_    ,							'''w'''       ) as f:
												f.write(A_    ,							arcname=os.path.join('''main_dir'''    ,							os.path.basename(A_       )       )       )
												f.write(A_    ,							arcname=os.path.join('''main_dir'''    ,							os.path.basename(A_       )       )       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Any    ,							A_							:					Union[str, Any]    ,							A_							:					str       ):
						snake_case       :     List[Any] =    tmp_path_factory.mktemp('''data'''       ) / '''dataset.ext.zip'''
						with zipfile.ZipFile(A_    ,							'''w'''       ) as f:
												f.write(A_    ,							arcname=os.path.basename('''unsupported.ext'''       )       )
												f.write(A_    ,							arcname=os.path.basename('''unsupported_2.ext'''       )       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Union[str, Any]       ):
						snake_case       :     Any =    '''\n'''.join(['''First''', '''Second\u2029with Unicode new line''', '''Third''']       )
						snake_case       :     Optional[int] =    str(tmp_path_factory.mktemp('''data'''       ) / '''dataset_with_unicode_new_lines.txt'''       )
						with open(A_    ,							'''w'''    ,							encoding='''utf-8'''       ) as f:
												f.write(A_       )
						return path

@pytest.fixture(scope='''session'''       )
def      A (       ):
						return os.path.join('''tests'''    ,							'''features'''    ,							'''data'''    ,							'''test_image_rgb.jpg'''       )

@pytest.fixture(scope='''session'''       )
def      A (       ):
						return os.path.join('''tests'''    ,							'''features'''    ,							'''data'''    ,							'''test_audio_44100.wav'''       )

@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Optional[Any]    ,							A_							:					Optional[Any]       ):
						snake_case       :     Union[str, Any] =    tmp_path_factory.mktemp('''data'''       ) / '''dataset.img.zip'''
						with zipfile.ZipFile(A_    ,							'''w'''       ) as f:
												f.write(A_    ,							arcname=os.path.basename(A_       )       )
												f.write(A_    ,							arcname=os.path.basename(A_       ).replace('''.jpg'''    ,							'''2.jpg'''       )       )
						return path





@pytest.fixture(scope='''session'''       )
def      A (       A_							:					Optional[Any]       ):
						snake_case       :     Dict =    tmp_path_factory.mktemp('''data_dir'''       )

						(data_dir / "subdir").mkdir()
						with open(data_dir / '''subdir''' / '''train.txt'''    ,							'''w'''       ) as f:
												f.write('''foo\n''' * 10       )
						with open(data_dir / '''subdir''' / '''test.txt'''    ,							'''w'''       ) as f:
												f.write('''bar\n''' * 10       )
						# hidden file
						with open(data_dir / '''subdir''' / '''.test.txt'''    ,							'''w'''       ) as f:
												f.write('''bar\n''' * 10       )

						# hidden directory
						(data_dir / ".subdir").mkdir()
						with open(data_dir / '''.subdir''' / '''train.txt'''    ,							'''w'''       ) as f:
												f.write('''foo\n''' * 10       )
						with open(data_dir / '''.subdir''' / '''test.txt'''    ,							'''w'''       ) as f:
												f.write('''bar\n''' * 10       )

						return data_dir

