import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_albert import AlbertTokenizer
else:
  SCREAMING_SNAKE_CASE			:						Any				  =		None

SCREAMING_SNAKE_CASE			:						str				  =		logging.get_logger(__name__)
SCREAMING_SNAKE_CASE			:						Optional[Any]				  =		{"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE			:						Optional[Any]				  =		{
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json""",
    },
}

SCREAMING_SNAKE_CASE			:						Any				  =		{
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

SCREAMING_SNAKE_CASE			:						Optional[int]				  =		"""▁"""
class  A_			(							a_			):
  _SCREAMING_SNAKE_CASE     					=      VOCAB_FILES_NAMES
  _SCREAMING_SNAKE_CASE     					=      PRETRAINED_VOCAB_FILES_MAP
  _SCREAMING_SNAKE_CASE     					=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _SCREAMING_SNAKE_CASE     					=      AlbertTokenizer

  def __init__(		self			:   List[Any]  ,				__SCREAMING_SNAKE_CASE			:   Union[str, Any]=None  ,				__SCREAMING_SNAKE_CASE			:   int=None  ,				__SCREAMING_SNAKE_CASE			:   Dict=True  ,				__SCREAMING_SNAKE_CASE			:   Tuple=True  ,				__SCREAMING_SNAKE_CASE			:   Dict=False  ,				__SCREAMING_SNAKE_CASE			:   Dict="[CLS]"  ,				__SCREAMING_SNAKE_CASE			:   Union[str, Any]="[SEP]"  ,				__SCREAMING_SNAKE_CASE			:   Dict="<unk>"  ,				__SCREAMING_SNAKE_CASE			:   str="[SEP]"  ,				__SCREAMING_SNAKE_CASE			:   Any="<pad>"  ,				__SCREAMING_SNAKE_CASE			:   List[str]="[CLS]"  ,				__SCREAMING_SNAKE_CASE			:   Optional[int]="[MASK]"  ,				**__SCREAMING_SNAKE_CASE			:   Tuple  ,				):
   # Mask token behave like a normal word, i.e. include the space before it and
   # is included in the raw text, there should be a match in a non-normalized sentence.
   __a							=	(
       AddedToken(__SCREAMING_SNAKE_CASE  ,				lstrip=__SCREAMING_SNAKE_CASE  ,				rstrip=__SCREAMING_SNAKE_CASE  ,				normalized=__SCREAMING_SNAKE_CASE							)
       if isinstance(__SCREAMING_SNAKE_CASE  ,				__SCREAMING_SNAKE_CASE							)
       else mask_token
   )

   super().__init__(
       __SCREAMING_SNAKE_CASE  ,				tokenizer_file=__SCREAMING_SNAKE_CASE  ,				do_lower_case=__SCREAMING_SNAKE_CASE  ,				remove_space=__SCREAMING_SNAKE_CASE  ,				keep_accents=__SCREAMING_SNAKE_CASE  ,				bos_token=__SCREAMING_SNAKE_CASE  ,				eos_token=__SCREAMING_SNAKE_CASE  ,				unk_token=__SCREAMING_SNAKE_CASE  ,				sep_token=__SCREAMING_SNAKE_CASE  ,				pad_token=__SCREAMING_SNAKE_CASE  ,				cls_token=__SCREAMING_SNAKE_CASE  ,				mask_token=__SCREAMING_SNAKE_CASE  ,				**__SCREAMING_SNAKE_CASE  ,				)

   __a							=	do_lower_case
   __a							=	remove_space
   __a							=	keep_accents
   __a							=	vocab_file
   __a							=	False if not self.vocab_file else True

  def 		_UpperCAmelCase (		self			:   List[str]  ,				__SCREAMING_SNAKE_CASE			:   List[int]  ,				__SCREAMING_SNAKE_CASE			:   Optional[List[int]] = None							):
   __a							=	[self.sep_token_id]
   __a							=	[self.cls_token_id]
   if token_ids_a is None:
    return cls + token_ids_a + sep
   return cls + token_ids_a + sep + token_ids_a + sep

  def 		_UpperCAmelCase (		self			:   Any  ,				__SCREAMING_SNAKE_CASE			:   List[int]  ,				__SCREAMING_SNAKE_CASE			:   Optional[List[int]] = None							):
   __a							=	[self.sep_token_id]
   __a							=	[self.cls_token_id]

   if token_ids_a is None:
    return len(cls + token_ids_a + sep							) * [0]
   return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]

  def 		_UpperCAmelCase (		self			:   Union[str, Any]  ,				__SCREAMING_SNAKE_CASE			:   str  ,				__SCREAMING_SNAKE_CASE			:   Optional[str] = None							):
   if not self.can_save_slow_tokenizer:
    raise ValueError(
        "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
        "tokenizer."							)

   if not os.path.isdir(__SCREAMING_SNAKE_CASE							):
    logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
    return
   __a							=	os.path.join(
       __SCREAMING_SNAKE_CASE  ,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)

   if os.path.abspath(self.vocab_file							) != os.path.abspath(__SCREAMING_SNAKE_CASE							):
    copyfile(self.vocab_file  ,				__SCREAMING_SNAKE_CASE							)

   return (out_vocab_file,)



import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config



@pytest.fixture(scope="session"       )
def 	__A		(      ):



 """simple docstring"""


 __a							=	10
 __a							=	datasets.Features(
     {
         "tokens": datasets.Sequence(datasets.Value("string"       )       ),
         "labels": datasets.Sequence(datasets.ClassLabel(names=["negative", "positive"]       )       ),
         "answers": datasets.Sequence(
             {
                 "text": datasets.Value("string"       ),
                 "answer_start": datasets.Value("int32"       ),
             }       ),
         "id": datasets.Value("int64"       ),
     }       )
 __a							=	datasets.Dataset.from_dict(
     {
         "tokens": [["foo"] * 5] * n,
         "labels": [[1] * 5] * n,
         "answers": [{"answer_start": [97], "text": ["1976"]}] * 10,
         "id": list(range(_A       )       ),
     }				,					features=_A				,					)
 return dataset



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "file.arrow"       )
 dataset.map(cache_file_name=_A       )
 return filename


# FILE_CONTENT + files


SCREAMING_SNAKE_CASE			:						Tuple				  =		"""\
    Text data.
    Second line of data."""



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt"
 __a							=	FILE_CONTENT
 with open(_A				,					"w"       ) as f:
  f.write(_A       )
 return filename



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 import bza

 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.bz2"
 __a							=	bytes(_A				,					"utf-8"       )
 with bza.open(_A				,					"wb"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 import gzip

 __a							=	str(tmp_path_factory.mktemp("data"       ) / "file.txt.gz"       )
 __a							=	bytes(_A				,					"utf-8"       )
 with gzip.open(_A				,					"wb"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 if datasets.config.LZ4_AVAILABLE:
  import lza.frame

  __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.lz4"
  __a							=	bytes(_A				,					"utf-8"       )
  with lza.frame.open(_A				,					"wb"       ) as f:
   f.write(_A       )
  return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 if datasets.config.PY7ZR_AVAILABLE:
  import pyazr

  __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.7z"
  with pyazr.SevenZipFile(_A				,					"w"       ) as archive:
   archive.write(_A				,					arcname=os.path.basename(_A       )       )
  return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import tarfile

 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.tar"
 with tarfile.TarFile(_A				,					"w"       ) as f:
  f.add(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 import lzma

 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.xz"
 __a							=	bytes(_A				,					"utf-8"       )
 with lzma.open(_A				,					"wb"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import zipfile

 __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 if datasets.config.ZSTANDARD_AVAILABLE:
  import zstandard as zstd

  __a							=	tmp_path_factory.mktemp("data"       ) / "file.txt.zst"
  __a							=	bytes(_A				,					"utf-8"       )
  with zstd.open(_A				,					"wb"       ) as f:
   f.write(_A       )
  return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "file.xml"
 __a							=	textwrap.dedent(
     "\\n    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n    <tmx version=\"1.4\">\n      <header segtype=\"sentence\" srclang=\"ca\" />\n      <body>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>"       )
 with open(_A				,					"w"       ) as f:
  f.write(_A       )
 return filename


SCREAMING_SNAKE_CASE			:						Any				  =		[
    {"""col_1""": """0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """3""", """col_2""": 3, """col_3""": 3.0},
]
SCREAMING_SNAKE_CASE			:						Optional[int]				  =		[
    {"""col_1""": """4""", """col_2""": 4, """col_3""": 4.0},
    {"""col_1""": """5""", """col_2""": 5, """col_3""": 5.0},
]
SCREAMING_SNAKE_CASE			:						Any				  =		{
    """col_1""": ["""0""", """1""", """2""", """3"""],
    """col_2""": [0, 1, 2, 3],
    """col_3""": [0.0, 1.0, 2.0, 3.0],
}

SCREAMING_SNAKE_CASE			:						int				  =		[
    {"""col_3""": 0.0, """col_1""": """0""", """col_2""": 0},
    {"""col_3""": 1.0, """col_1""": """1""", """col_2""": 1},
]

SCREAMING_SNAKE_CASE			:						List[Any]				  =		[
    {"""col_1""": """s0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """s1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """s2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """s3""", """col_2""": 3, """col_3""": 3.0},
]



@pytest.fixture(scope="session"       )
def 	__A		(      ):



 """simple docstring"""


 return DATA_DICT_OF_LISTS



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	datasets.Dataset.from_dict(_A       )
 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.arrow"       )
 dataset.map(cache_file_name=_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.sqlite"       )
 with contextlib.closing(sqlitea.connect(_A       )       ) as con:
  __a							=	con.cursor()
  cur.execute("CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)"       )
  for item in DATA:
   cur.execute("INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)"				,					tuple(item.values()       )       )
  con.commit()
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.csv"       )
 with open(_A				,					"w"				,					newline=""       ) as f:
  __a							=	csv.DictWriter(_A				,					fieldnames=["col_1", "col_2", "col_3"]       )
  writer.writeheader()
  for item in DATA:
   writer.writerow(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset2.csv"       )
 with open(_A				,					"w"				,					newline=""       ) as f:
  __a							=	csv.DictWriter(_A				,					fieldnames=["col_1", "col_2", "col_3"]       )
  writer.writeheader()
  for item in DATA:
   writer.writerow(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import bza

 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.csv.bz2"
 with open(_A				,					"rb"       ) as f:
  __a							=	f.read()
 # data = bytes(FILE_CONTENT, "utf-8")
 with bza.open(_A				,					"wb"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.csv.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
  f.write(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.csv.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(csv_path.replace(".csv"				,					".CSV"       )       )       )
  f.write(_A				,					arcname=os.path.basename(csva_path.replace(".csv"				,					".CSV"       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_with_dir.csv.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.parquet"       )
 __a							=	pa.schema(
     {
         "col_1": pa.string(),
         "col_2": pa.intaa(),
         "col_3": pa.floataa(),
     }       )
 with open(_A				,					"wb"       ) as f:
  __a							=	pq.ParquetWriter(_A				,					schema=_A       )
  __a							=	pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(_A       )       )] for k in DATA[0]}				,					schema=_A       )
  writer.write_table(_A       )
  writer.close()
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.json"       )
 __a							=	{"data": DATA}
 with open(_A				,					"w"       ) as f:
  json.dump(_A				,					_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.json"       )
 __a							=	{"data": DATA_DICT_OF_LISTS}
 with open(_A				,					"w"       ) as f:
  json.dump(_A				,					_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.jsonl"       )
 with open(_A				,					"w"       ) as f:
  for item in DATA:
   f.write(json.dumps(_A       ) + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset2.jsonl"       )
 with open(_A				,					"w"       ) as f:
  for item in DATA:
   f.write(json.dumps(_A       ) + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset_312.jsonl"       )
 with open(_A				,					"w"       ) as f:
  for item in DATA_312:
   f.write(json.dumps(_A       ) + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset-str.jsonl"       )
 with open(_A				,					"w"       ) as f:
  for item in DATA_STR:
   f.write(json.dumps(_A       ) + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import gzip

 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.txt.gz"       )
 with open(_A				,					"rb"       ) as orig_file:
  with gzip.open(_A				,					"wb"       ) as zipped_file:
   zipped_file.writelines(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 import gzip

 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.jsonl.gz"       )
 with open(_A				,					"rb"       ) as orig_file:
  with gzip.open(_A				,					"wb"       ) as zipped_file:
   zipped_file.writelines(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.jsonl.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
  f.write(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_nested.jsonl.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.join("nested"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_with_dir.jsonl.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.jsonl.tar"
 with tarfile.TarFile(_A				,					"w"       ) as f:
  f.add(_A				,					arcname=os.path.basename(_A       )       )
  f.add(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_nested.jsonl.tar"
 with tarfile.TarFile(_A				,					"w"       ) as f:
  f.add(_A				,					arcname=os.path.join("nested"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	["0", "1", "2", "3"]
 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset.txt"       )
 with open(_A				,					"w"       ) as f:
  for item in data:
   f.write(item + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	["0", "1", "2", "3"]
 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset2.txt"       )
 with open(_A				,					"w"       ) as f:
  for item in data:
   f.write(item + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	["0", "1", "2", "3"]
 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.abc"
 with open(_A				,					"w"       ) as f:
  for item in data:
   f.write(item + "\n"       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.text.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
  f.write(_A				,					arcname=os.path.basename(_A       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset_with_dir.text.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
  f.write(_A				,					arcname=os.path.join("main_dir"				,					os.path.basename(_A       )       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.ext.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename("unsupported.ext"       )       )
  f.write(_A				,					arcname=os.path.basename("unsupported_2.ext"       )       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	"\n".join(["First", "Second\u2029with Unicode new line", "Third"]       )
 __a							=	str(tmp_path_factory.mktemp("data"       ) / "dataset_with_unicode_new_lines.txt"       )
 with open(_A				,					"w"				,					encoding="utf-8"       ) as f:
  f.write(_A       )
 return path



@pytest.fixture(scope="session"       )
def 	__A		(      ):



 """simple docstring"""


 return os.path.join("tests"				,					"features"				,					"data"				,					"test_image_rgb.jpg"       )



@pytest.fixture(scope="session"       )
def 	__A		(      ):



 """simple docstring"""


 return os.path.join("tests"				,					"features"				,					"data"				,					"test_audio_44100.wav"       )



@pytest.fixture(scope="session"       )
def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data"       ) / "dataset.img.zip"
 with zipfile.ZipFile(_A				,					"w"       ) as f:
  f.write(_A				,					arcname=os.path.basename(_A       )       )
  f.write(_A				,					arcname=os.path.basename(_A       ).replace(".jpg"				,					"2.jpg"       )       )
 return path






@pytest.fixture(scope="session"       )
def 	__A		(      _A       ):



 """simple docstring"""


 __a							=	tmp_path_factory.mktemp("data_dir"       )

 (data_dir / "subdir").mkdir()
 with open(data_dir / "subdir" / "train.txt"				,					"w"       ) as f:
  f.write("foo\n" * 10       )
 with open(data_dir / "subdir" / "test.txt"				,					"w"       ) as f:
  f.write("bar\n" * 10       )
 # hidden file
 with open(data_dir / "subdir" / ".test.txt"				,					"w"       ) as f:
  f.write("bar\n" * 10       )

 # hidden directory
 (data_dir / ".subdir").mkdir()
 with open(data_dir / ".subdir" / "train.txt"				,					"w"       ) as f:
  f.write("foo\n" * 10       )
 with open(data_dir / ".subdir" / "test.txt"				,					"w"       ) as f:
  f.write("bar\n" * 10       )

 return data_dir


