



import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase						  =		logging.get_logger(__name__)

_lowercase						  =		"▁"

_lowercase						  =		{
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
}

_lowercase						  =		{
    "vocab_file": {
        "facebook/s2t-small-librispeech-asr": (
            "https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json"
        ),
    },
    "spm_file": {
        "facebook/s2t-small-librispeech-asr": (
            "https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model"
        )
    },
}

_lowercase						  =		{
    "facebook/s2t-small-librispeech-asr": 1024,
}

_lowercase						  =		["pt", "fr", "ru", "nl", "ro", "it", "es", "de"]

_lowercase						  =		{"mustc": MUSTC_LANGS}



class     _UpperCAmelCase			(       A__     ):
     UpperCamelCase__		    =				VOCAB_FILES_NAMES
     UpperCamelCase__		    =				PRETRAINED_VOCAB_FILES_MAP
     UpperCamelCase__		    =				MAX_MODEL_INPUT_SIZES
     UpperCamelCase__		    =				['''input_ids''', '''attention_mask''']

     UpperCamelCase__		    =				[]




     def __init__(				self ,						a__ ,						a__ ,						a__="<s>" ,						a__="</s>" ,						a__="<pad>" ,						a__="<unk>" ,						a__=False ,						a__=False ,						a__=None ,						a__=None ,						a__ = None ,						**a__ ,						):
         A__		 =					{} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=a__ ,						eos_token=a__ ,						unk_token=a__ ,						pad_token=a__ ,						do_upper_case=a__ ,						do_lower_case=a__ ,						tgt_lang=a__ ,						lang_codes=a__ ,						sp_model_kwargs=self.sp_model_kwargs ,						**a__ ,						)
         A__		 =					do_upper_case
         A__		 =					do_lower_case

         A__		 =					load_json(a__)
         A__		 =					{v: k for k, v in self.encoder.items()}
         A__		 =					spm_file
         A__		 =					load_spm(a__ ,						self.sp_model_kwargs)

         if lang_codes is not None:
             A__		 =					lang_codes
             A__		 =					LANGUAGES[lang_codes]
             A__		 =					[F"<lang:{lang}>" for lang in self.langs]
             A__		 =					{lang: self.sp_model.PieceToId(F"<lang:{lang}>") for lang in self.langs}

             A__		 =					self.lang_tokens
             A__		 =					tgt_lang if tgt_lang is not None else self.langs[0]

             self.set_tgt_lang_special_tokens(self._tgt_lang)
         else:
             A__		 =					{}




     @property
     def 		snake_case_		(				self):
         return len(self.encoder)




     @property
     def 		snake_case_		(				self):
         return self._tgt_lang




     @tgt_lang.setter
     def 		snake_case_		(				self ,						a__):
         A__		 =					new_tgt_lang
         self.set_tgt_lang_special_tokens(a__)




     def 		snake_case_		(				self ,						a__):
         A__		 =					self.lang_code_to_id[tgt_lang]
         A__		 =					[lang_code_id]




     def 		snake_case_		(				self ,						a__):
         return self.sp_model.encode(a__ ,						out_type=a__)




     def 		snake_case_		(				self ,						a__):
         return self.encoder.get(a__ ,						self.encoder[self.unk_token])




     def 		snake_case_		(				self ,						a__):
         return self.decoder.get(a__ ,						self.unk_token)




     def 		snake_case_		(				self ,						a__):
         A__		 =					[]
         A__		 =					''''''
         for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                 A__		 =					self.sp_model.decode(a__)
                 out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
                 A__		 =					[]
             else:
                 current_sub_tokens.append(a__)
         A__		 =					self.sp_model.decode(a__)
         out_string += decoded.upper() if self.do_upper_case else decoded
         return out_string.strip()




     def 		snake_case_		(				self ,						a__ ,						a__=None):
         if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + [self.eos_token_id]
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]




     def 		snake_case_		(				self ,						a__ ,						a__ = None ,						a__ = False):

         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=a__ ,						token_ids_a=a__ ,						already_has_special_tokens=a__)

         A__		 =					[1] * len(self.prefix_tokens)
         A__		 =					[1]
         if token_ids_a is None:
             return prefix_ones + ([0] * len(a__)) + suffix_ones
         return prefix_ones + ([0] * len(a__)) + ([0] * len(a__)) + suffix_ones




     def 		snake_case_		(				self):
         A__		 =					self.encoder.copy()
         vocab.update(self.added_tokens_encoder)
         return vocab




     def __getstate__(				self):
         A__		 =					self.__dict__.copy()
         A__		 =					None
         return state




     def __setstate__(				self ,						a__):
         A__		 =					d

         # for backward compatibility
         if not hasattr(self ,						'''sp_model_kwargs'''):
             A__		 =					{}

         A__		 =					load_spm(self.spm_file ,						self.sp_model_kwargs)




     def 		snake_case_		(				self ,						a__ ,						a__ = None):
         A__		 =					Path(a__)
         assert save_dir.is_dir(), F"{save_directory} should be a directory"
         A__		 =					save_dir / (
             (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
         )
         A__		 =					save_dir / (
             (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
         )

         save_json(self.encoder ,						a__)

         if os.path.abspath(self.spm_file) != os.path.abspath(a__) and os.path.isfile(self.spm_file):
             copyfile(self.spm_file ,						a__)
         elif not os.path.isfile(self.spm_file):
             with open(a__ ,						'''wb''') as fi:
                 A__		 =					self.sp_model.serialized_model_proto()
                 fi.write(a__)

         return (str(a__), str(a__))







def 		lowerCAmelCase__    (					UpperCamelCase_				:       str ,       UpperCamelCase_				:       Dict[str, Any]	)->  sentencepiece.SentencePieceProcessor:
    A__		 =					sentencepiece.SentencePieceProcessor(**UpperCamelCase_	)
    spm.Load(str(UpperCamelCase_	)	)
    return spm



def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  Union[Dict, List]:
    with open(UpperCamelCase_ ,       '''r'''	) as f:
        return json.load(UpperCamelCase_	)



def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       str	)->  None:
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        json.dump(UpperCamelCase_ ,       UpperCamelCase_ ,       indent=2	)





import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  str:
    A__		 =					1_0
    A__		 =					datasets.Features(
        {
            '''tokens''': datasets.Sequence(datasets.Value('''string'''	)	),
            '''labels''': datasets.Sequence(datasets.ClassLabel(names=['''negative''', '''positive''']	)	),
            '''answers''': datasets.Sequence(
                {
                    '''text''': datasets.Value('''string'''	),
                    '''answer_start''': datasets.Value('''int32'''	),
                }	),
            '''id''': datasets.Value('''int64'''	),
        }	)
    A__		 =					datasets.Dataset.from_dict(
        {
            '''tokens''': [['''foo'''] * 5] * n,
            '''labels''': [[1] * 5] * n,
            '''answers''': [{'''answer_start''': [9_7], '''text''': ['''1976''']}] * 1_0,
            '''id''': list(range(UpperCamelCase_	)	),
        } ,       features=UpperCamelCase_ ,       )
    return dataset



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       int	)->  Optional[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''file.arrow'''	)
    dataset.map(cache_file_name=UpperCamelCase_	)
    return filename


# FILE_CONTENT + files


_lowercase						  =		"\\n    Text data.\n    Second line of data."



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  List[Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt'''
    A__		 =					FILE_CONTENT
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_	)
    return filename



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  Optional[Any]:
    import bza

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.bz2'''
    A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
    with bza.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any]	)->  int:
    import gzip

    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''file.txt.gz'''	)
    A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
    with gzip.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  Any:
    if datasets.config.LZ4_AVAILABLE:
        import lza.frame

        A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.lz4'''
        A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
        with lza.frame.open(UpperCamelCase_ ,       '''wb'''	) as f:
            f.write(UpperCamelCase_	)
        return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[Any] ,       UpperCamelCase_				:       Tuple	)->  int:
    if datasets.config.PY7ZR_AVAILABLE:
        import pyazr

        A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.7z'''
        with pyazr.SevenZipFile(UpperCamelCase_ ,       '''w'''	) as archive:
            archive.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Tuple ,       UpperCamelCase_				:       int	)->  Optional[Any]:
    import tarfile

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.tar'''
    with tarfile.TarFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.add(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  str:
    import lzma

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.xz'''
    A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
    with lzma.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int] ,       UpperCamelCase_				:       List[str]	)->  List[str]:
    import zipfile

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Tuple	)->  str:
    if datasets.config.ZSTANDARD_AVAILABLE:
        import zstandard as zstd

        A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.zst'''
        A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
        with zstd.open(UpperCamelCase_ ,       '''wb'''	) as f:
            f.write(UpperCamelCase_	)
        return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  int:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.xml'''
    A__		 =					textwrap.dedent(
        '''\
    <?xml version="1.0" encoding="UTF-8" ?>
    <tmx version="1.4">
      <header segtype="sentence" srclang="ca" />
      <body>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang="en"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang="en"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang="en"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang="en"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang="en"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_	)
    return filename


_lowercase						  =		[
    {"col_1": "0", "col_2": 0, "col_3": 0.0},
    {"col_1": "1", "col_2": 1, "col_3": 1.0},
    {"col_1": "2", "col_2": 2, "col_3": 2.0},
    {"col_1": "3", "col_2": 3, "col_3": 3.0},
]
_lowercase						  =		[
    {"col_1": "4", "col_2": 4, "col_3": 4.0},
    {"col_1": "5", "col_2": 5, "col_3": 5.0},
]
_lowercase						  =		{
    "col_1": ["0", "1", "2", "3"],
    "col_2": [0, 1, 2, 3],
    "col_3": [0.0, 1.0, 2.0, 3.0],
}

_lowercase						  =		[
    {"col_3": 0.0, "col_1": "0", "col_2": 0},
    {"col_3": 1.0, "col_1": "1", "col_2": 1},
]

_lowercase						  =		[
    {"col_1": "s0", "col_2": 0, "col_3": 0.0},
    {"col_1": "s1", "col_2": 1, "col_3": 1.0},
    {"col_1": "s2", "col_2": 2, "col_3": 2.0},
    {"col_1": "s3", "col_2": 3, "col_3": 3.0},
]



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  str:
    return DATA_DICT_OF_LISTS



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[Any]	)->  List[str]:
    A__		 =					datasets.Dataset.from_dict(UpperCamelCase_	)
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.arrow'''	)
    dataset.map(cache_file_name=UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  List[str]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.sqlite'''	)
    with contextlib.closing(sqlitea.connect(UpperCamelCase_	)	) as con:
        A__		 =					con.cursor()
        cur.execute('''CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'''	)
        for item in DATA:
            cur.execute('''INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)''' ,       tuple(item.values()	)	)
        con.commit()
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any]	)->  Tuple:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.csv'''	)
    with open(UpperCamelCase_ ,       '''w''' ,       newline=''''''	) as f:
        A__		 =					csv.DictWriter(UpperCamelCase_ ,       fieldnames=['''col_1''', '''col_2''', '''col_3''']	)
        writer.writeheader()
        for item in DATA:
            writer.writerow(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  List[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset2.csv'''	)
    with open(UpperCamelCase_ ,       '''w''' ,       newline=''''''	) as f:
        A__		 =					csv.DictWriter(UpperCamelCase_ ,       fieldnames=['''col_1''', '''col_2''', '''col_3''']	)
        writer.writeheader()
        for item in DATA:
            writer.writerow(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       Optional[int]	)->  List[str]:
    import bza

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.csv.bz2'''
    with open(UpperCamelCase_ ,       '''rb'''	) as f:
        A__		 =					f.read()
    # data = bytes(FILE_CONTENT, "utf-8")
    with bza.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str ,       UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       Optional[int]	)->  str:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.csv.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       str ,       UpperCamelCase_				:       Any	)->  List[Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.csv.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(csv_path.replace('''.csv''' ,       '''.CSV'''	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(csva_path.replace('''.csv''' ,       '''.CSV'''	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str ,       UpperCamelCase_				:       str ,       UpperCamelCase_				:       Optional[Any]	)->  Tuple:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_with_dir.csv.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  Optional[int]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.parquet'''	)
    A__		 =					pa.schema(
        {
            '''col_1''': pa.string(),
            '''col_2''': pa.intaa(),
            '''col_3''': pa.floataa(),
        }	)
    with open(UpperCamelCase_ ,       '''wb'''	) as f:
        A__		 =					pq.ParquetWriter(UpperCamelCase_ ,       schema=UpperCamelCase_	)
        A__		 =					pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(UpperCamelCase_	)	)] for k in DATA[0]} ,       schema=UpperCamelCase_	)
        writer.write_table(UpperCamelCase_	)
        writer.close()
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  str:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.json'''	)
    A__		 =					{'''data''': DATA}
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        json.dump(UpperCamelCase_ ,       UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  Optional[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.json'''	)
    A__		 =					{'''data''': DATA_DICT_OF_LISTS}
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        json.dump(UpperCamelCase_ ,       UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  List[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  Tuple:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset2.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int	)->  Optional[int]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset_312.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA_312:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int	)->  Union[str, Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset-str.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA_STR:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any] ,       UpperCamelCase_				:       str	)->  List[str]:
    import gzip

    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.txt.gz'''	)
    with open(UpperCamelCase_ ,       '''rb'''	) as orig_file:
        with gzip.open(UpperCamelCase_ ,       '''wb'''	) as zipped_file:
            zipped_file.writelines(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Any ,       UpperCamelCase_				:       Optional[Any]	)->  List[Any]:
    import gzip

    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl.gz'''	)
    with open(UpperCamelCase_ ,       '''rb'''	) as orig_file:
        with gzip.open(UpperCamelCase_ ,       '''wb'''	) as zipped_file:
            zipped_file.writelines(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       List[str]	)->  Optional[int]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       Optional[int]	)->  Union[str, Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_nested.jsonl.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''nested''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       Any	)->  Any:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_with_dir.jsonl.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict ,       UpperCamelCase_				:       Any ,       UpperCamelCase_				:       str	)->  Tuple:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl.tar'''
    with tarfile.TarFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.add(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.add(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int] ,       UpperCamelCase_				:       int ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       List[str]	)->  str:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_nested.jsonl.tar'''
    with tarfile.TarFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.add(UpperCamelCase_ ,       arcname=os.path.join('''nested''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Any	)->  List[str]:
    A__		 =					['''0''', '''1''', '''2''', '''3''']
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.txt'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in data:
            f.write(item + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int	)->  Dict:
    A__		 =					['''0''', '''1''', '''2''', '''3''']
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset2.txt'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in data:
            f.write(item + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any]	)->  Union[str, Any]:
    A__		 =					['''0''', '''1''', '''2''', '''3''']
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.abc'''
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in data:
            f.write(item + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       Any ,       UpperCamelCase_				:       Optional[Any]	)->  List[str]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.text.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Any ,       UpperCamelCase_				:       List[Any] ,       UpperCamelCase_				:       Optional[int]	)->  Tuple:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_with_dir.text.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       Tuple ,       UpperCamelCase_				:       Tuple	)->  Any:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.ext.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename('''unsupported.ext'''	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename('''unsupported_2.ext'''	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  Any:
    A__		 =					'''\n'''.join(['''First''', '''Second\u2029with Unicode new line''', '''Third''']	)
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset_with_unicode_new_lines.txt'''	)
    with open(UpperCamelCase_ ,       '''w''' ,       encoding='''utf-8'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  Tuple:
    return os.path.join('''tests''' ,       '''features''' ,       '''data''' ,       '''test_image_rgb.jpg'''	)



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  Dict:
    return os.path.join('''tests''' ,       '''features''' ,       '''data''' ,       '''test_audio_44100.wav'''	)



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any] ,       UpperCamelCase_				:       Union[str, Any]	)->  List[Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.img.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	).replace('''.jpg''' ,       '''2.jpg'''	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any]	)->  str:
    A__		 =					tmp_path_factory.mktemp('''data_dir'''	)

    (data_dir / "subdir").mkdir()
    with open(data_dir / '''subdir''' / '''train.txt''' ,       '''w'''	) as f:
        f.write('''foo\n''' * 1_0	)
    with open(data_dir / '''subdir''' / '''test.txt''' ,       '''w'''	) as f:
        f.write('''bar\n''' * 1_0	)
    # hidden file
    with open(data_dir / '''subdir''' / '''.test.txt''' ,       '''w'''	) as f:
        f.write('''bar\n''' * 1_0	)

    # hidden directory
    (data_dir / ".subdir").mkdir()
    with open(data_dir / '''.subdir''' / '''train.txt''' ,       '''w'''	) as f:
        f.write('''foo\n''' * 1_0	)
    with open(data_dir / '''.subdir''' / '''test.txt''' ,       '''w'''	) as f:
        f.write('''bar\n''' * 1_0	)

    return data_dir
