
"""simple docstring"""







import importlib.metadata
import warnings
from copy import deepcopy

from packaging import version

from ..utils import logging
from .import_utils import is_accelerate_available, is_bitsandbytes_available


if is_bitsandbytes_available():
       import bitsandbytes as bnb
       import torch
       import torch.nn as nn

       from ..pytorch_utils import ConvaD

if is_accelerate_available():
       from accelerate import init_empty_weights
       from accelerate.utils import find_tied_parameters

lowercase_         =       logging.get_logger(__name__)




def       A_      (       lowercase    ,			lowercase    ,			lowercase    ,			lowercase=None    ,			lowercase=None   )  ->				Tuple:


     """simple docstring"""







     if "." in tensor_name:
          UpperCAmelCase_    :     str            =	tensor_name.split("""."""   )
          for split in splits[:-1]:
               UpperCAmelCase_    :     Optional[Any]            =	getattr(lowercase    ,			lowercase   )
               if new_module is None:
                    raise ValueError(f'''{module} has no attribute {split}.'''   )
               UpperCAmelCase_    :     str            =	new_module
          UpperCAmelCase_    :     str            =	splits[-1]

     if tensor_name not in module._parameters and tensor_name not in module._buffers:
          raise ValueError(f'''{module} does not have a parameter or a buffer named {tensor_name}.'''   )
     UpperCAmelCase_    :     Dict            =	tensor_name in module._buffers
     UpperCAmelCase_    :     Any            =	getattr(lowercase    ,			lowercase   )

     if old_value.device == torch.device("""meta"""   ) and device not in ["meta", torch.device("""meta"""   )] and value is None:
          raise ValueError(f'''{tensor_name} is on the meta device, we need a `value` to put in on {device}.'''   )

     UpperCAmelCase_    :     List[Any]            =	False
     UpperCAmelCase_    :     List[Any]            =	False
     if is_buffer or not is_bitsandbytes_available():
          UpperCAmelCase_    :     Tuple            =	False
          UpperCAmelCase_    :     Tuple            =	False
     else:
          UpperCAmelCase_    :     List[Any]            =	hasattr(bnb.nn    ,			"""Params4bit"""   ) and isinstance(module._parameters[tensor_name]    ,			bnb.nn.Paramsabit   )
          UpperCAmelCase_    :     str            =	isinstance(module._parameters[tensor_name]    ,			bnb.nn.IntaParams   )

     if is_abit or is_abit:
          UpperCAmelCase_    :     Dict            =	module._parameters[tensor_name]
          if param.device.type != "cuda":
               if value is None:
                    UpperCAmelCase_    :     int            =	old_value.to(lowercase   )
               elif isinstance(lowercase    ,			torch.Tensor   ):
                    UpperCAmelCase_    :     Optional[int]            =	value.to("""cpu"""   )
                    if value.dtype == torch.inta:
                         UpperCAmelCase_    :     Dict            =	version.parse(importlib.metadata.version("""bitsandbytes"""   )   ) > version.parse(
                             """0.37.2"""   )
                         if not is_abit_serializable:
                              raise ValueError(
                                  """Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. """
                                  """Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`."""   )
               else:
                    UpperCAmelCase_    :     List[Any]            =	torch.tensor(lowercase    ,			device="""cpu"""   )

               # Support models using `Conv1D` in place of `nn.Linear` (e.g. gpt2) by transposing the weight matrix prior to quantization.
               # Since weights are saved in the correct "orientation", we skip transposing when loading.
               if issubclass(module.source_cls    ,			lowercase   ) and fpaa_statistics is None:
                    UpperCAmelCase_    :     Dict            =	new_value.T

               UpperCAmelCase_    :     int            =	old_value.__dict__
               if is_abit:
                    UpperCAmelCase_    :     List[Any]            =	bnb.nn.IntaParams(lowercase    ,			requires_grad=lowercase    ,			**lowercase   ).to(lowercase   )
               elif is_abit:
                    UpperCAmelCase_    :     Optional[Any]            =	bnb.nn.Paramsabit(lowercase    ,			requires_grad=lowercase    ,			**lowercase   ).to(lowercase   )

               UpperCAmelCase_    :     Union[str, Any]            =	new_value
               if fpaa_statistics is not None:
                    setattr(module.weight    ,			"""SCB"""    ,			fpaa_statistics.to(lowercase   )   )

     else:
          if value is None:
               UpperCAmelCase_    :     Dict            =	old_value.to(lowercase   )
          elif isinstance(lowercase    ,			torch.Tensor   ):
               UpperCAmelCase_    :     int            =	value.to(lowercase   )
          else:
               UpperCAmelCase_    :     str            =	torch.tensor(lowercase    ,			device=lowercase   )

          if is_buffer:
               UpperCAmelCase_    :     Optional[Any]            =	new_value
          else:
               UpperCAmelCase_    :     int            =	nn.Parameter(lowercase    ,			requires_grad=old_value.requires_grad   )
               UpperCAmelCase_    :     Dict            =	new_value




def       A_      (       lowercase    ,			lowercase=None    ,			lowercase=None    ,			lowercase=None    ,			lowercase=False   )  ->				List[Any]:


     """simple docstring"""







     for name, module in model.named_children():
          if current_key_name is None:
               UpperCAmelCase_    :     List[Any]            =	[]
          current_key_name.append(lowercase   )

          if (isinstance(lowercase    ,			nn.Linear   ) or isinstance(lowercase    ,			lowercase   )) and name not in modules_to_not_convert:
               # Check if the current key is not in the `modules_to_not_convert`
               if not any(key in """.""".join(lowercase   ) for key in modules_to_not_convert   ):
                    with init_empty_weights():
                         if isinstance(lowercase    ,			lowercase   ):
                              UpperCAmelCase_       ,UpperCAmelCase_    :     str            =	module.weight.shape
                         else:
                              UpperCAmelCase_    :     Any            =	module.in_features
                              UpperCAmelCase_    :     List[str]            =	module.out_features

                         if quantization_config.quantization_method() == "llm_int8":
                              UpperCAmelCase_    :     Optional[Any]            =	bnb.nn.LinearabitLt(
                                  lowercase    ,			lowercase    ,			module.bias is not None    ,			has_fpaa_weights=quantization_config.llm_inta_has_fpaa_weight    ,			threshold=quantization_config.llm_inta_threshold    ,			)
                              UpperCAmelCase_    :     str            =	True
                         else:
                              if (
                                  quantization_config.llm_inta_skip_modules is not None
                                  and name in quantization_config.llm_inta_skip_modules
                              ):
                                   pass
                              else:
                                   UpperCAmelCase_    :     int            =	bnb.nn.Linearabit(
                                       lowercase    ,			lowercase    ,			module.bias is not None    ,			quantization_config.bnb_abit_compute_dtype    ,			compress_statistics=quantization_config.bnb_abit_use_double_quant    ,			quant_type=quantization_config.bnb_abit_quant_type    ,			)
                                   UpperCAmelCase_    :     Union[str, Any]            =	True
                    # Store the module class in case we need to transpose the weight later
                         UpperCAmelCase_    :     int            =	type(lowercase   )
                         # Force requires grad to False to avoid unexpected errors
                         model._modules[name].requires_grad_(lowercase   )
          if len(list(module.children()   )   ) > 0:
               UpperCAmelCase_       ,UpperCAmelCase_    :     Tuple            =	_replace_with_bnb_linear(
                   lowercase    ,			lowercase    ,			lowercase    ,			lowercase    ,			has_been_replaced=lowercase    ,			)
          # Remove the last key for recursion
          current_key_name.pop(-1   )
     return model, has_been_replaced




def       A_      (       lowercase    ,			lowercase=None    ,			lowercase=None    ,			lowercase=None   )  ->				Dict:


     """simple docstring"""







     UpperCAmelCase_    :     Optional[int]            =	["""lm_head"""] if modules_to_not_convert is None else modules_to_not_convert
     UpperCAmelCase_       ,UpperCAmelCase_    :     Dict            =	_replace_with_bnb_linear(
         lowercase    ,			lowercase    ,			lowercase    ,			lowercase   )

     if not has_been_replaced:
          logger.warning(
              """You are loading your model in 8bit or 4bit but no linear modules were found in your model."""
              """ Please double check your model architecture, or submit an issue on github if you think this is"""
              """ a bug."""   )

     return model




def       A_      (       *lowercase    ,			**lowercase   )  ->				Dict:


     """simple docstring"""







     warnings.warn(
         """`replace_8bit_linear` will be deprecated in a future version, please use `replace_with_bnb_linear` instead"""    ,			lowercase    ,			)
     return replace_with_bnb_linear(*lowercase    ,			**lowercase   )




def       A_      (       *lowercase    ,			**lowercase   )  ->				Optional[Any]:


     """simple docstring"""







     warnings.warn(
         """`set_module_8bit_tensor_to_device` will be deprecated in a future version, please use `set_module_quantized_tensor_to_device` instead"""    ,			lowercase    ,			)
     return set_module_quantized_tensor_to_device(*lowercase    ,			**lowercase   )




def       A_      (       lowercase   )  ->				Any:


     """simple docstring"""







     UpperCAmelCase_    :     Union[str, Any]            =	deepcopy(lowercase   )  # this has 0 cost since it is done inside `init_empty_weights` context manager`
     tied_model.tie_weights()

     UpperCAmelCase_    :     Tuple            =	find_tied_parameters(lowercase   )
     # For compatibility with Accelerate < 0.18
     if isinstance(lowercase    ,			lowercase   ):
          UpperCAmelCase_    :     Tuple            =	sum(list(tied_params.values()   )    ,			[]   ) + list(tied_params.keys()   )
     else:
          UpperCAmelCase_    :     Union[str, Any]            =	sum(lowercase    ,			[]   )
     UpperCAmelCase_    :     Union[str, Any]            =	len(lowercase   ) > 0

     # Check if it is a base model
     UpperCAmelCase_    :     Optional[int]            =	not hasattr(lowercase    ,			model.base_model_prefix   )

     # Ignore this for base models (BertModel, GPT2Model, etc.)
     if (not has_tied_params) and is_base_model:
          return []

     # otherwise they have an attached head
     UpperCAmelCase_    :     str            =	list(model.named_children()   )
     UpperCAmelCase_    :     List[Any]            =	[list_modules[-1][0]]

     # add last module together with tied weights
     UpperCAmelCase_    :     Optional[Any]            =	set(lowercase   ) - set(lowercase   )
     UpperCAmelCase_    :     str            =	list(set(lowercase   )   ) + list(lowercase   )

     # remove ".weight" from the keys
     UpperCAmelCase_    :     int            =	[""".weight""", """.bias"""]
     UpperCAmelCase_    :     Any            =	[]
     for name in list_untouched:
          for name_to_remove in names_to_remove:
               if name_to_remove in name:
                    UpperCAmelCase_    :     List[Any]            =	name.replace(lowercase    ,			""""""   )
          filtered_module_names.append(lowercase   )

     return filtered_module_names



"""simple docstring"""







import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config




@pytest.fixture(scope="""session"""   )
def       A_      (       )  ->				str:


     """simple docstring"""







     UpperCAmelCase_    :     int            =	10
     UpperCAmelCase_    :     Optional[Any]            =	datasets.Features(
         {
             """tokens""": datasets.Sequence(datasets.Value("""string"""   )   ),
             """labels""": datasets.Sequence(datasets.ClassLabel(names=["""negative""", """positive"""]   )   ),
             """answers""": datasets.Sequence(
                 {
                     """text""": datasets.Value("""string"""   ),
                     """answer_start""": datasets.Value("""int32"""   ),
                 }   ),
             """id""": datasets.Value("""int64"""   ),
         }   )
     UpperCAmelCase_    :     List[Any]            =	datasets.Dataset.from_dict(
         {
             """tokens""": [["""foo"""] * 5] * n,
             """labels""": [[1] * 5] * n,
             """answers""": [{"""answer_start""": [97], """text""": ["""1976"""]}] * 10,
             """id""": list(range(lowercase   )   ),
         }    ,			features=lowercase    ,			)
     return dataset




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase   )  ->				int:


     """simple docstring"""







     UpperCAmelCase_    :     Optional[Any]            =	str(tmp_path_factory.mktemp("""data"""   ) / """file.arrow"""   )
     dataset.map(cache_file_name=lowercase   )
     return filename


# FILE_CONTENT + files


lowercase_         =       "\\n    Text data.\n    Second line of data."




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Optional[Any]:


     """simple docstring"""







     UpperCAmelCase_    :     int            =	tmp_path_factory.mktemp("""data"""   ) / """file.txt"""
     UpperCAmelCase_    :     Tuple            =	FILE_CONTENT
     with open(lowercase    ,			"""w"""   ) as f:
          f.write(lowercase   )
     return filename




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Dict:


     """simple docstring"""







     import bza

     UpperCAmelCase_    :     int            =	tmp_path_factory.mktemp("""data"""   ) / """file.txt.bz2"""
     UpperCAmelCase_    :     List[str]            =	bytes(lowercase    ,			"""utf-8"""   )
     with bza.open(lowercase    ,			"""wb"""   ) as f:
          f.write(lowercase   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				List[str]:


     """simple docstring"""







     import gzip

     UpperCAmelCase_    :     List[str]            =	str(tmp_path_factory.mktemp("""data"""   ) / """file.txt.gz"""   )
     UpperCAmelCase_    :     Optional[Any]            =	bytes(lowercase    ,			"""utf-8"""   )
     with gzip.open(lowercase    ,			"""wb"""   ) as f:
          f.write(lowercase   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Union[str, Any]:


     """simple docstring"""







     if datasets.config.LZ4_AVAILABLE:
          import lza.frame

          UpperCAmelCase_    :     Dict            =	tmp_path_factory.mktemp("""data"""   ) / """file.txt.lz4"""
          UpperCAmelCase_    :     str            =	bytes(lowercase    ,			"""utf-8"""   )
          with lza.frame.open(lowercase    ,			"""wb"""   ) as f:
               f.write(lowercase   )
          return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase   )  ->				Optional[Any]:


     """simple docstring"""







     if datasets.config.PY7ZR_AVAILABLE:
          import pyazr

          UpperCAmelCase_    :     List[str]            =	tmp_path_factory.mktemp("""data"""   ) / """file.txt.7z"""
          with pyazr.SevenZipFile(lowercase    ,			"""w"""   ) as archive:
               archive.write(lowercase    ,			arcname=os.path.basename(lowercase   )   )
          return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase   )  ->				Union[str, Any]:


     """simple docstring"""







     import tarfile

     UpperCAmelCase_    :     Optional[int]            =	tmp_path_factory.mktemp("""data"""   ) / """file.txt.tar"""
     with tarfile.TarFile(lowercase    ,			"""w"""   ) as f:
          f.add(lowercase    ,			arcname=os.path.basename(lowercase   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				int:


     """simple docstring"""







     import lzma

     UpperCAmelCase_    :     List[Any]            =	tmp_path_factory.mktemp("""data"""   ) / """file.txt.xz"""
     UpperCAmelCase_    :     List[str]            =	bytes(lowercase    ,			"""utf-8"""   )
     with lzma.open(lowercase    ,			"""wb"""   ) as f:
          f.write(lowercase   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase   )  ->				Any:


     """simple docstring"""







     import zipfile

     UpperCAmelCase_    :     Any            =	tmp_path_factory.mktemp("""data"""   ) / """file.txt.zip"""
     with zipfile.ZipFile(lowercase    ,			"""w"""   ) as f:
          f.write(lowercase    ,			arcname=os.path.basename(lowercase   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Dict:


     """simple docstring"""







     if datasets.config.ZSTANDARD_AVAILABLE:
          import zstandard as zstd

          UpperCAmelCase_    :     List[str]            =	tmp_path_factory.mktemp("""data"""   ) / """file.txt.zst"""
          UpperCAmelCase_    :     Optional[int]            =	bytes(lowercase    ,			"""utf-8"""   )
          with zstd.open(lowercase    ,			"""wb"""   ) as f:
               f.write(lowercase   )
          return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Dict:


     """simple docstring"""







     UpperCAmelCase_    :     int            =	tmp_path_factory.mktemp("""data"""   ) / """file.xml"""
     UpperCAmelCase_    :     int            =	textwrap.dedent(
         """\
    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>
    <tmx version=\"1.4\">
      <header segtype=\"sentence\" srclang=\"ca\" />
      <body>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>"""   )
     with open(lowercase    ,			"""w"""   ) as f:
          f.write(lowercase   )
     return filename


lowercase_         =       [
    {"col_1": "0", "col_2": 0, "col_3": 0.0},
    {"col_1": "1", "col_2": 1, "col_3": 1.0},
    {"col_1": "2", "col_2": 2, "col_3": 2.0},
    {"col_1": "3", "col_2": 3, "col_3": 3.0},
]
lowercase_         =       [
    {"col_1": "4", "col_2": 4, "col_3": 4.0},
    {"col_1": "5", "col_2": 5, "col_3": 5.0},
]
lowercase_         =       {
    "col_1": ["0", "1", "2", "3"],
    "col_2": [0, 1, 2, 3],
    "col_3": [0.0, 1.0, 2.0, 3.0],
}

lowercase_         =       [
    {"col_3": 0.0, "col_1": "0", "col_2": 0},
    {"col_3": 1.0, "col_1": "1", "col_2": 1},
]

lowercase_         =       [
    {"col_1": "s0", "col_2": 0, "col_3": 0.0},
    {"col_1": "s1", "col_2": 1, "col_3": 1.0},
    {"col_1": "s2", "col_2": 2, "col_3": 2.0},
    {"col_1": "s3", "col_2": 3, "col_3": 3.0},
]




@pytest.fixture(scope="""session"""   )
def       A_      (       )  ->				Union[str, Any]:


     """simple docstring"""







     return DATA_DICT_OF_LISTS




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				int:


     """simple docstring"""







     UpperCAmelCase_    :     str            =	datasets.Dataset.from_dict(lowercase   )
     UpperCAmelCase_    :     List[str]            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset.arrow"""   )
     dataset.map(cache_file_name=lowercase   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				List[Any]:


     """simple docstring"""







     UpperCAmelCase_    :     Dict            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset.sqlite"""   )
     with contextlib.closing(sqlitea.connect(lowercase   )   ) as con:
          UpperCAmelCase_    :     Tuple            =	con.cursor()
          cur.execute("""CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)"""   )
          for item in DATA:
               cur.execute("""INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)"""    ,			tuple(item.values()   )   )
          con.commit()
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Optional[int]:


     """simple docstring"""







     UpperCAmelCase_    :     Union[str, Any]            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset.csv"""   )
     with open(lowercase    ,			"""w"""    ,			newline=""""""   ) as f:
          UpperCAmelCase_    :     Tuple            =	csv.DictWriter(lowercase    ,			fieldnames=["""col_1""", """col_2""", """col_3"""]   )
          writer.writeheader()
          for item in DATA:
               writer.writerow(lowercase   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				str:


     """simple docstring"""







     UpperCAmelCase_    :     Tuple            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset2.csv"""   )
     with open(lowercase    ,			"""w"""    ,			newline=""""""   ) as f:
          UpperCAmelCase_    :     List[Any]            =	csv.DictWriter(lowercase    ,			fieldnames=["""col_1""", """col_2""", """col_3"""]   )
          writer.writeheader()
          for item in DATA:
               writer.writerow(lowercase   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase   )  ->				Optional[Any]:


     """simple docstring"""







     import bza

     UpperCAmelCase_    :     List[Any]            =	tmp_path_factory.mktemp("""data"""   ) / """dataset.csv.bz2"""
     with open(lowercase    ,			"""rb"""   ) as f:
          UpperCAmelCase_    :     Dict            =	f.read()
     # data = bytes(FILE_CONTENT, "utf-8")
     with bza.open(lowercase    ,			"""wb"""   ) as f:
          f.write(lowercase   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase    ,			lowercase   )  ->				List[Any]:


     """simple docstring"""







     UpperCAmelCase_    :     Optional[Any]            =	tmp_path_factory.mktemp("""data"""   ) / """dataset.csv.zip"""
     with zipfile.ZipFile(lowercase    ,			"""w"""   ) as f:
          f.write(lowercase    ,			arcname=os.path.basename(lowercase   )   )
          f.write(lowercase    ,			arcname=os.path.basename(lowercase   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase    ,			lowercase   )  ->				List[Any]:


     """simple docstring"""







     UpperCAmelCase_    :     Tuple            =	tmp_path_factory.mktemp("""data"""   ) / """dataset.csv.zip"""
     with zipfile.ZipFile(lowercase    ,			"""w"""   ) as f:
          f.write(lowercase    ,			arcname=os.path.basename(csv_path.replace(""".csv"""    ,			""".CSV"""   )   )   )
          f.write(lowercase    ,			arcname=os.path.basename(csva_path.replace(""".csv"""    ,			""".CSV"""   )   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase    ,			lowercase   )  ->				Optional[int]:


     """simple docstring"""







     UpperCAmelCase_    :     Union[str, Any]            =	tmp_path_factory.mktemp("""data"""   ) / """dataset_with_dir.csv.zip"""
     with zipfile.ZipFile(lowercase    ,			"""w"""   ) as f:
          f.write(lowercase    ,			arcname=os.path.join("""main_dir"""    ,			os.path.basename(lowercase   )   )   )
          f.write(lowercase    ,			arcname=os.path.join("""main_dir"""    ,			os.path.basename(lowercase   )   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Any:


     """simple docstring"""







     UpperCAmelCase_    :     Union[str, Any]            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset.parquet"""   )
     UpperCAmelCase_    :     Dict            =	pa.schema(
         {
             """col_1""": pa.string(),
             """col_2""": pa.intaa(),
             """col_3""": pa.floataa(),
         }   )
     with open(lowercase    ,			"""wb"""   ) as f:
          UpperCAmelCase_    :     List[str]            =	pq.ParquetWriter(lowercase    ,			schema=lowercase   )
          UpperCAmelCase_    :     Any            =	pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(lowercase   )   )] for k in DATA[0]}    ,			schema=lowercase   )
          writer.write_table(lowercase   )
          writer.close()
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Tuple:


     """simple docstring"""







     UpperCAmelCase_    :     Union[str, Any]            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset.json"""   )
     UpperCAmelCase_    :     Union[str, Any]            =	{"""data""": DATA}
     with open(lowercase    ,			"""w"""   ) as f:
          json.dump(lowercase    ,			lowercase   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Optional[int]:


     """simple docstring"""







     UpperCAmelCase_    :     Union[str, Any]            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset.json"""   )
     UpperCAmelCase_    :     Optional[int]            =	{"""data""": DATA_DICT_OF_LISTS}
     with open(lowercase    ,			"""w"""   ) as f:
          json.dump(lowercase    ,			lowercase   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				str:


     """simple docstring"""







     UpperCAmelCase_    :     List[str]            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset.jsonl"""   )
     with open(lowercase    ,			"""w"""   ) as f:
          for item in DATA:
               f.write(json.dumps(lowercase   ) + """\n"""   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Union[str, Any]:


     """simple docstring"""







     UpperCAmelCase_    :     str            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset2.jsonl"""   )
     with open(lowercase    ,			"""w"""   ) as f:
          for item in DATA:
               f.write(json.dumps(lowercase   ) + """\n"""   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Optional[Any]:


     """simple docstring"""







     UpperCAmelCase_    :     int            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset_312.jsonl"""   )
     with open(lowercase    ,			"""w"""   ) as f:
          for item in DATA_312:
               f.write(json.dumps(lowercase   ) + """\n"""   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Optional[int]:


     """simple docstring"""







     UpperCAmelCase_    :     Dict            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset-str.jsonl"""   )
     with open(lowercase    ,			"""w"""   ) as f:
          for item in DATA_STR:
               f.write(json.dumps(lowercase   ) + """\n"""   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase   )  ->				Dict:


     """simple docstring"""







     import gzip

     UpperCAmelCase_    :     Dict            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset.txt.gz"""   )
     with open(lowercase    ,			"""rb"""   ) as orig_file:
          with gzip.open(lowercase    ,			"""wb"""   ) as zipped_file:
               zipped_file.writelines(lowercase   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase   )  ->				str:


     """simple docstring"""







     import gzip

     UpperCAmelCase_    :     Optional[Any]            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset.jsonl.gz"""   )
     with open(lowercase    ,			"""rb"""   ) as orig_file:
          with gzip.open(lowercase    ,			"""wb"""   ) as zipped_file:
               zipped_file.writelines(lowercase   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase    ,			lowercase   )  ->				Optional[int]:


     """simple docstring"""







     UpperCAmelCase_    :     Tuple            =	tmp_path_factory.mktemp("""data"""   ) / """dataset.jsonl.zip"""
     with zipfile.ZipFile(lowercase    ,			"""w"""   ) as f:
          f.write(lowercase    ,			arcname=os.path.basename(lowercase   )   )
          f.write(lowercase    ,			arcname=os.path.basename(lowercase   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase    ,			lowercase    ,			lowercase   )  ->				List[Any]:


     """simple docstring"""







     UpperCAmelCase_    :     List[Any]            =	tmp_path_factory.mktemp("""data"""   ) / """dataset_nested.jsonl.zip"""
     with zipfile.ZipFile(lowercase    ,			"""w"""   ) as f:
          f.write(lowercase    ,			arcname=os.path.join("""nested"""    ,			os.path.basename(lowercase   )   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase    ,			lowercase   )  ->				Optional[Any]:


     """simple docstring"""







     UpperCAmelCase_    :     Optional[int]            =	tmp_path_factory.mktemp("""data"""   ) / """dataset_with_dir.jsonl.zip"""
     with zipfile.ZipFile(lowercase    ,			"""w"""   ) as f:
          f.write(lowercase    ,			arcname=os.path.join("""main_dir"""    ,			os.path.basename(lowercase   )   )   )
          f.write(lowercase    ,			arcname=os.path.join("""main_dir"""    ,			os.path.basename(lowercase   )   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase    ,			lowercase   )  ->				int:


     """simple docstring"""







     UpperCAmelCase_    :     int            =	tmp_path_factory.mktemp("""data"""   ) / """dataset.jsonl.tar"""
     with tarfile.TarFile(lowercase    ,			"""w"""   ) as f:
          f.add(lowercase    ,			arcname=os.path.basename(lowercase   )   )
          f.add(lowercase    ,			arcname=os.path.basename(lowercase   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase    ,			lowercase    ,			lowercase   )  ->				Union[str, Any]:


     """simple docstring"""







     UpperCAmelCase_    :     Optional[int]            =	tmp_path_factory.mktemp("""data"""   ) / """dataset_nested.jsonl.tar"""
     with tarfile.TarFile(lowercase    ,			"""w"""   ) as f:
          f.add(lowercase    ,			arcname=os.path.join("""nested"""    ,			os.path.basename(lowercase   )   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Tuple:


     """simple docstring"""







     UpperCAmelCase_    :     Tuple            =	["""0""", """1""", """2""", """3"""]
     UpperCAmelCase_    :     str            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset.txt"""   )
     with open(lowercase    ,			"""w"""   ) as f:
          for item in data:
               f.write(item + """\n"""   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				str:


     """simple docstring"""







     UpperCAmelCase_    :     Tuple            =	["""0""", """1""", """2""", """3"""]
     UpperCAmelCase_    :     int            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset2.txt"""   )
     with open(lowercase    ,			"""w"""   ) as f:
          for item in data:
               f.write(item + """\n"""   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				int:


     """simple docstring"""







     UpperCAmelCase_    :     Dict            =	["""0""", """1""", """2""", """3"""]
     UpperCAmelCase_    :     Optional[int]            =	tmp_path_factory.mktemp("""data"""   ) / """dataset.abc"""
     with open(lowercase    ,			"""w"""   ) as f:
          for item in data:
               f.write(item + """\n"""   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase    ,			lowercase   )  ->				int:


     """simple docstring"""







     UpperCAmelCase_    :     Any            =	tmp_path_factory.mktemp("""data"""   ) / """dataset.text.zip"""
     with zipfile.ZipFile(lowercase    ,			"""w"""   ) as f:
          f.write(lowercase    ,			arcname=os.path.basename(lowercase   )   )
          f.write(lowercase    ,			arcname=os.path.basename(lowercase   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase    ,			lowercase   )  ->				List[Any]:


     """simple docstring"""







     UpperCAmelCase_    :     Tuple            =	tmp_path_factory.mktemp("""data"""   ) / """dataset_with_dir.text.zip"""
     with zipfile.ZipFile(lowercase    ,			"""w"""   ) as f:
          f.write(lowercase    ,			arcname=os.path.join("""main_dir"""    ,			os.path.basename(lowercase   )   )   )
          f.write(lowercase    ,			arcname=os.path.join("""main_dir"""    ,			os.path.basename(lowercase   )   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase    ,			lowercase   )  ->				str:


     """simple docstring"""







     UpperCAmelCase_    :     List[Any]            =	tmp_path_factory.mktemp("""data"""   ) / """dataset.ext.zip"""
     with zipfile.ZipFile(lowercase    ,			"""w"""   ) as f:
          f.write(lowercase    ,			arcname=os.path.basename("""unsupported.ext"""   )   )
          f.write(lowercase    ,			arcname=os.path.basename("""unsupported_2.ext"""   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Union[str, Any]:


     """simple docstring"""







     UpperCAmelCase_    :     Tuple            =	"""\n""".join(["""First""", """Second\u2029with Unicode new line""", """Third"""]   )
     UpperCAmelCase_    :     List[Any]            =	str(tmp_path_factory.mktemp("""data"""   ) / """dataset_with_unicode_new_lines.txt"""   )
     with open(lowercase    ,			"""w"""    ,			encoding="""utf-8"""   ) as f:
          f.write(lowercase   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       )  ->				Optional[int]:


     """simple docstring"""







     return os.path.join("""tests"""    ,			"""features"""    ,			"""data"""    ,			"""test_image_rgb.jpg"""   )




@pytest.fixture(scope="""session"""   )
def       A_      (       )  ->				Tuple:


     """simple docstring"""







     return os.path.join("""tests"""    ,			"""features"""    ,			"""data"""    ,			"""test_audio_44100.wav"""   )




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase    ,			lowercase   )  ->				int:


     """simple docstring"""







     UpperCAmelCase_    :     Dict            =	tmp_path_factory.mktemp("""data"""   ) / """dataset.img.zip"""
     with zipfile.ZipFile(lowercase    ,			"""w"""   ) as f:
          f.write(lowercase    ,			arcname=os.path.basename(lowercase   )   )
          f.write(lowercase    ,			arcname=os.path.basename(lowercase   ).replace(""".jpg"""    ,			"""2.jpg"""   )   )
     return path




@pytest.fixture(scope="""session"""   )
def       A_      (       lowercase   )  ->				Dict:


     """simple docstring"""







     UpperCAmelCase_    :     Optional[int]            =	tmp_path_factory.mktemp("""data_dir"""   )

     (data_dir / "subdir").mkdir()
     with open(data_dir / """subdir""" / """train.txt"""    ,			"""w"""   ) as f:
          f.write("""foo\n""" * 10   )
     with open(data_dir / """subdir""" / """test.txt"""    ,			"""w"""   ) as f:
          f.write("""bar\n""" * 10   )
     # hidden file
     with open(data_dir / """subdir""" / """.test.txt"""    ,			"""w"""   ) as f:
          f.write("""bar\n""" * 10   )

     # hidden directory
     (data_dir / ".subdir").mkdir()
     with open(data_dir / """.subdir""" / """train.txt"""    ,			"""w"""   ) as f:
          f.write("""foo\n""" * 10   )
     with open(data_dir / """.subdir""" / """test.txt"""    ,			"""w"""   ) as f:
          f.write("""bar\n""" * 10   )

     return data_dir

