



import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  )							->	Union[str, Any]:







			"""simple docstring"""
			a__       :						str							=  10
			a__       :						Union[str, Any]							=  datasets.Features(
			    {
			        """tokens""": datasets.Sequence(datasets.Value("""string""")),
			        """labels""": datasets.Sequence(datasets.ClassLabel(names=["""negative""", """positive"""])),
			        """answers""": datasets.Sequence(
			            {
			                """text""": datasets.Value("""string"""),
			                """answer_start""": datasets.Value("""int32"""),
			            }),
			        """id""": datasets.Value("""int64"""),
			    })
			a__       :						List[str]							=  datasets.Dataset.from_dict(
			    {
			        """tokens""": [["""foo"""] * 5] * n,
			        """labels""": [[1] * 5] * n,
			        """answers""": [{"""answer_start""": [97], """text""": ["""1976"""]}] * 10,
			        """id""": list(range(_lowercase)),
			    }						,       features=_lowercase						,       )
			return dataset

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Any						,       _lowercase    :       List[str])							->	List[Any]:







			"""simple docstring"""
			a__       :						List[Any]							=  str(tmp_path_factory.mktemp("""data""") / """file.arrow""")
			dataset.map(cache_file_name=_lowercase)
			return filename


# FILE_CONTENT + files


_lowercase		:				List[str]      		="\\n    Text data.\n    Second line of data."

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Union[str, Any])							->	Optional[int]:







			"""simple docstring"""
			a__       :						Optional[Any]							=  tmp_path_factory.mktemp("""data""") / """file.txt"""
			a__       :						Any							=  FILE_CONTENT
			with open(_lowercase						,       """w""") as f:
						f.write(_lowercase)
			return filename

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Tuple)							->	Optional[Any]:







			"""simple docstring"""
			import bza

			a__       :						Optional[Any]							=  tmp_path_factory.mktemp("""data""") / """file.txt.bz2"""
			a__       :						Any							=  bytes(_lowercase						,       """utf-8""")
			with bza.open(_lowercase						,       """wb""") as f:
						f.write(_lowercase)
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       int)							->	int:







			"""simple docstring"""
			import gzip

			a__       :						Optional[Any]							=  str(tmp_path_factory.mktemp("""data""") / """file.txt.gz""")
			a__       :						Optional[Any]							=  bytes(_lowercase						,       """utf-8""")
			with gzip.open(_lowercase						,       """wb""") as f:
						f.write(_lowercase)
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       List[str])							->	int:







			"""simple docstring"""
			if datasets.config.LZ4_AVAILABLE:
						import lza.frame

						a__       :						Any							=  tmp_path_factory.mktemp("""data""") / """file.txt.lz4"""
						a__       :						Dict							=  bytes(_lowercase						,       """utf-8""")
						with lza.frame.open(_lowercase						,       """wb""") as f:
									f.write(_lowercase)
						return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Dict						,       _lowercase    :       str)							->	Tuple:







			"""simple docstring"""
			if datasets.config.PY7ZR_AVAILABLE:
						import pyazr

						a__       :						int							=  tmp_path_factory.mktemp("""data""") / """file.txt.7z"""
						with pyazr.SevenZipFile(_lowercase						,       """w""") as archive:
									archive.write(_lowercase						,       arcname=os.path.basename(_lowercase))
						return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Union[str, Any]						,       _lowercase    :       Union[str, Any])							->	List[str]:







			"""simple docstring"""
			import tarfile

			a__       :						Optional[Any]							=  tmp_path_factory.mktemp("""data""") / """file.txt.tar"""
			with tarfile.TarFile(_lowercase						,       """w""") as f:
						f.add(_lowercase						,       arcname=os.path.basename(_lowercase))
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[int])							->	Optional[int]:







			"""simple docstring"""
			import lzma

			a__       :						Optional[Any]							=  tmp_path_factory.mktemp("""data""") / """file.txt.xz"""
			a__       :						List[str]							=  bytes(_lowercase						,       """utf-8""")
			with lzma.open(_lowercase						,       """wb""") as f:
						f.write(_lowercase)
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Any						,       _lowercase    :       Dict)							->	Any:







			"""simple docstring"""
			import zipfile

			a__       :						Any							=  tmp_path_factory.mktemp("""data""") / """file.txt.zip"""
			with zipfile.ZipFile(_lowercase						,       """w""") as f:
						f.write(_lowercase						,       arcname=os.path.basename(_lowercase))
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[Any])							->	List[Any]:







			"""simple docstring"""
			if datasets.config.ZSTANDARD_AVAILABLE:
						import zstandard as zstd

						a__       :						Optional[Any]							=  tmp_path_factory.mktemp("""data""") / """file.txt.zst"""
						a__       :						Optional[int]							=  bytes(_lowercase						,       """utf-8""")
						with zstd.open(_lowercase						,       """wb""") as f:
									f.write(_lowercase)
						return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Tuple)							->	str:







			"""simple docstring"""
			a__       :						Dict							=  tmp_path_factory.mktemp("""data""") / """file.xml"""
			a__       :						List[Any]							=  textwrap.dedent(
			    """\
    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>
    <tmx version=\"1.4\">
      <header segtype=\"sentence\" srclang=\"ca\" />
      <body>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>""")
			with open(_lowercase						,       """w""") as f:
						f.write(_lowercase)
			return filename


_lowercase		:				Optional[int]      		=[
    {"col_1": "0", "col_2": 0, "col_3": 0.0},
    {"col_1": "1", "col_2": 1, "col_3": 1.0},
    {"col_1": "2", "col_2": 2, "col_3": 2.0},
    {"col_1": "3", "col_2": 3, "col_3": 3.0},
]
_lowercase		:				List[str]      		=[
    {"col_1": "4", "col_2": 4, "col_3": 4.0},
    {"col_1": "5", "col_2": 5, "col_3": 5.0},
]
_lowercase		:				int      		={
    "col_1": ["0", "1", "2", "3"],
    "col_2": [0, 1, 2, 3],
    "col_3": [0.0, 1.0, 2.0, 3.0],
}

_lowercase		:				Tuple      		=[
    {"col_3": 0.0, "col_1": "0", "col_2": 0},
    {"col_3": 1.0, "col_1": "1", "col_2": 1},
]

_lowercase		:				int      		=[
    {"col_1": "s0", "col_2": 0, "col_3": 0.0},
    {"col_1": "s1", "col_2": 1, "col_3": 1.0},
    {"col_1": "s2", "col_2": 2, "col_3": 2.0},
    {"col_1": "s3", "col_2": 3, "col_3": 3.0},
]

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  )							->	Optional[int]:







			"""simple docstring"""
			return DATA_DICT_OF_LISTS

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       List[Any])							->	List[str]:







			"""simple docstring"""
			a__       :						Tuple							=  datasets.Dataset.from_dict(_lowercase)
			a__       :						List[str]							=  str(tmp_path_factory.mktemp("""data""") / """dataset.arrow""")
			dataset.map(cache_file_name=_lowercase)
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       List[str])							->	str:







			"""simple docstring"""
			a__       :						Optional[int]							=  str(tmp_path_factory.mktemp("""data""") / """dataset.sqlite""")
			with contextlib.closing(sqlitea.connect(_lowercase)) as con:
						a__       :						Any							=  con.cursor()
						cur.execute("""CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)""")
						for item in DATA:
									cur.execute("""INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)"""						,       tuple(item.values()))
						con.commit()
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       int)							->	Union[str, Any]:







			"""simple docstring"""
			a__       :						str							=  str(tmp_path_factory.mktemp("""data""") / """dataset.csv""")
			with open(_lowercase						,       """w"""						,       newline="""""") as f:
						a__       :						List[Any]							=  csv.DictWriter(_lowercase						,       fieldnames=["""col_1""", """col_2""", """col_3"""])
						writer.writeheader()
						for item in DATA:
									writer.writerow(_lowercase)
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       str)							->	Dict:







			"""simple docstring"""
			a__       :						Dict							=  str(tmp_path_factory.mktemp("""data""") / """dataset2.csv""")
			with open(_lowercase						,       """w"""						,       newline="""""") as f:
						a__       :						str							=  csv.DictWriter(_lowercase						,       fieldnames=["""col_1""", """col_2""", """col_3"""])
						writer.writeheader()
						for item in DATA:
									writer.writerow(_lowercase)
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[int]						,       _lowercase    :       List[Any])							->	List[Any]:







			"""simple docstring"""
			import bza

			a__       :						Dict							=  tmp_path_factory.mktemp("""data""") / """dataset.csv.bz2"""
			with open(_lowercase						,       """rb""") as f:
						a__       :						Dict							=  f.read()
			# data = bytes(FILE_CONTENT, "utf-8")
			with bza.open(_lowercase						,       """wb""") as f:
						f.write(_lowercase)
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Any						,       _lowercase    :       Optional[int]						,       _lowercase    :       List[Any])							->	str:







			"""simple docstring"""
			a__       :						Dict							=  tmp_path_factory.mktemp("""data""") / """dataset.csv.zip"""
			with zipfile.ZipFile(_lowercase						,       """w""") as f:
						f.write(_lowercase						,       arcname=os.path.basename(_lowercase))
						f.write(_lowercase						,       arcname=os.path.basename(_lowercase))
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[int]						,       _lowercase    :       str						,       _lowercase    :       int)							->	Optional[Any]:







			"""simple docstring"""
			a__       :						str							=  tmp_path_factory.mktemp("""data""") / """dataset.csv.zip"""
			with zipfile.ZipFile(_lowercase						,       """w""") as f:
						f.write(_lowercase						,       arcname=os.path.basename(csv_path.replace(""".csv"""						,       """.CSV""")))
						f.write(_lowercase						,       arcname=os.path.basename(csva_path.replace(""".csv"""						,       """.CSV""")))
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Any						,       _lowercase    :       Any						,       _lowercase    :       List[str])							->	str:







			"""simple docstring"""
			a__       :						Optional[Any]							=  tmp_path_factory.mktemp("""data""") / """dataset_with_dir.csv.zip"""
			with zipfile.ZipFile(_lowercase						,       """w""") as f:
						f.write(_lowercase						,       arcname=os.path.join("""main_dir"""						,       os.path.basename(_lowercase)))
						f.write(_lowercase						,       arcname=os.path.join("""main_dir"""						,       os.path.basename(_lowercase)))
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Any)							->	str:







			"""simple docstring"""
			a__       :						int							=  str(tmp_path_factory.mktemp("""data""") / """dataset.parquet""")
			a__       :						Optional[Any]							=  pa.schema(
			    {
			        """col_1""": pa.string(),
			        """col_2""": pa.intaa(),
			        """col_3""": pa.floataa(),
			    })
			with open(_lowercase						,       """wb""") as f:
						a__       :						List[str]							=  pq.ParquetWriter(_lowercase						,       schema=_lowercase)
						a__       :						Dict							=  pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(_lowercase))] for k in DATA[0]}						,       schema=_lowercase)
						writer.write_table(_lowercase)
						writer.close()
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[int])							->	Dict:







			"""simple docstring"""
			a__       :						Union[str, Any]							=  str(tmp_path_factory.mktemp("""data""") / """dataset.json""")
			a__       :						List[str]							=  {"""data""": DATA}
			with open(_lowercase						,       """w""") as f:
						json.dump(_lowercase						,       _lowercase)
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       str)							->	Dict:







			"""simple docstring"""
			a__       :						Optional[int]							=  str(tmp_path_factory.mktemp("""data""") / """dataset.json""")
			a__       :						Union[str, Any]							=  {"""data""": DATA_DICT_OF_LISTS}
			with open(_lowercase						,       """w""") as f:
						json.dump(_lowercase						,       _lowercase)
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[Any])							->	Tuple:







			"""simple docstring"""
			a__       :						List[Any]							=  str(tmp_path_factory.mktemp("""data""") / """dataset.jsonl""")
			with open(_lowercase						,       """w""") as f:
						for item in DATA:
									f.write(json.dumps(_lowercase) + """\n""")
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Any)							->	int:







			"""simple docstring"""
			a__       :						str							=  str(tmp_path_factory.mktemp("""data""") / """dataset2.jsonl""")
			with open(_lowercase						,       """w""") as f:
						for item in DATA:
									f.write(json.dumps(_lowercase) + """\n""")
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[Any])							->	int:







			"""simple docstring"""
			a__       :						Tuple							=  str(tmp_path_factory.mktemp("""data""") / """dataset_312.jsonl""")
			with open(_lowercase						,       """w""") as f:
						for item in DATA_312:
									f.write(json.dumps(_lowercase) + """\n""")
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[Any])							->	List[Any]:







			"""simple docstring"""
			a__       :						Optional[Any]							=  str(tmp_path_factory.mktemp("""data""") / """dataset-str.jsonl""")
			with open(_lowercase						,       """w""") as f:
						for item in DATA_STR:
									f.write(json.dumps(_lowercase) + """\n""")
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       int						,       _lowercase    :       Any)							->	str:







			"""simple docstring"""
			import gzip

			a__       :						int							=  str(tmp_path_factory.mktemp("""data""") / """dataset.txt.gz""")
			with open(_lowercase						,       """rb""") as orig_file:
						with gzip.open(_lowercase						,       """wb""") as zipped_file:
									zipped_file.writelines(_lowercase)
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       str						,       _lowercase    :       List[Any])							->	Optional[int]:







			"""simple docstring"""
			import gzip

			a__       :						Any							=  str(tmp_path_factory.mktemp("""data""") / """dataset.jsonl.gz""")
			with open(_lowercase						,       """rb""") as orig_file:
						with gzip.open(_lowercase						,       """wb""") as zipped_file:
									zipped_file.writelines(_lowercase)
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[int]						,       _lowercase    :       Tuple						,       _lowercase    :       Optional[int])							->	List[str]:







			"""simple docstring"""
			a__       :						str							=  tmp_path_factory.mktemp("""data""") / """dataset.jsonl.zip"""
			with zipfile.ZipFile(_lowercase						,       """w""") as f:
						f.write(_lowercase						,       arcname=os.path.basename(_lowercase))
						f.write(_lowercase						,       arcname=os.path.basename(_lowercase))
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[Any]						,       _lowercase    :       Any						,       _lowercase    :       Any						,       _lowercase    :       str)							->	List[Any]:







			"""simple docstring"""
			a__       :						str							=  tmp_path_factory.mktemp("""data""") / """dataset_nested.jsonl.zip"""
			with zipfile.ZipFile(_lowercase						,       """w""") as f:
						f.write(_lowercase						,       arcname=os.path.join("""nested"""						,       os.path.basename(_lowercase)))
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[Any]						,       _lowercase    :       Dict						,       _lowercase    :       str)							->	Any:







			"""simple docstring"""
			a__       :						Union[str, Any]							=  tmp_path_factory.mktemp("""data""") / """dataset_with_dir.jsonl.zip"""
			with zipfile.ZipFile(_lowercase						,       """w""") as f:
						f.write(_lowercase						,       arcname=os.path.join("""main_dir"""						,       os.path.basename(_lowercase)))
						f.write(_lowercase						,       arcname=os.path.join("""main_dir"""						,       os.path.basename(_lowercase)))
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Tuple						,       _lowercase    :       Optional[Any]						,       _lowercase    :       int)							->	Dict:







			"""simple docstring"""
			a__       :						Optional[Any]							=  tmp_path_factory.mktemp("""data""") / """dataset.jsonl.tar"""
			with tarfile.TarFile(_lowercase						,       """w""") as f:
						f.add(_lowercase						,       arcname=os.path.basename(_lowercase))
						f.add(_lowercase						,       arcname=os.path.basename(_lowercase))
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Tuple						,       _lowercase    :       List[Any]						,       _lowercase    :       Dict						,       _lowercase    :       Any)							->	Optional[int]:







			"""simple docstring"""
			a__       :						Dict							=  tmp_path_factory.mktemp("""data""") / """dataset_nested.jsonl.tar"""
			with tarfile.TarFile(_lowercase						,       """w""") as f:
						f.add(_lowercase						,       arcname=os.path.join("""nested"""						,       os.path.basename(_lowercase)))
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Union[str, Any])							->	str:







			"""simple docstring"""
			a__       :						Optional[int]							=  ["""0""", """1""", """2""", """3"""]
			a__       :						List[str]							=  str(tmp_path_factory.mktemp("""data""") / """dataset.txt""")
			with open(_lowercase						,       """w""") as f:
						for item in data:
									f.write(item + """\n""")
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Tuple)							->	int:







			"""simple docstring"""
			a__       :						Optional[Any]							=  ["""0""", """1""", """2""", """3"""]
			a__       :						Tuple							=  str(tmp_path_factory.mktemp("""data""") / """dataset2.txt""")
			with open(_lowercase						,       """w""") as f:
						for item in data:
									f.write(item + """\n""")
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       List[str])							->	Optional[int]:







			"""simple docstring"""
			a__       :						Optional[int]							=  ["""0""", """1""", """2""", """3"""]
			a__       :						Optional[int]							=  tmp_path_factory.mktemp("""data""") / """dataset.abc"""
			with open(_lowercase						,       """w""") as f:
						for item in data:
									f.write(item + """\n""")
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Dict						,       _lowercase    :       Tuple						,       _lowercase    :       Optional[Any])							->	Union[str, Any]:







			"""simple docstring"""
			a__       :						int							=  tmp_path_factory.mktemp("""data""") / """dataset.text.zip"""
			with zipfile.ZipFile(_lowercase						,       """w""") as f:
						f.write(_lowercase						,       arcname=os.path.basename(_lowercase))
						f.write(_lowercase						,       arcname=os.path.basename(_lowercase))
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Union[str, Any]						,       _lowercase    :       int						,       _lowercase    :       int)							->	Union[str, Any]:







			"""simple docstring"""
			a__       :						Dict							=  tmp_path_factory.mktemp("""data""") / """dataset_with_dir.text.zip"""
			with zipfile.ZipFile(_lowercase						,       """w""") as f:
						f.write(_lowercase						,       arcname=os.path.join("""main_dir"""						,       os.path.basename(_lowercase)))
						f.write(_lowercase						,       arcname=os.path.join("""main_dir"""						,       os.path.basename(_lowercase)))
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[int]						,       _lowercase    :       Tuple						,       _lowercase    :       Any)							->	str:







			"""simple docstring"""
			a__       :						Optional[Any]							=  tmp_path_factory.mktemp("""data""") / """dataset.ext.zip"""
			with zipfile.ZipFile(_lowercase						,       """w""") as f:
						f.write(_lowercase						,       arcname=os.path.basename("""unsupported.ext"""))
						f.write(_lowercase						,       arcname=os.path.basename("""unsupported_2.ext"""))
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Union[str, Any])							->	List[Any]:







			"""simple docstring"""
			a__       :						Optional[int]							=  """\n""".join(["""First""", """Second\u2029with Unicode new line""", """Third"""])
			a__       :						Dict							=  str(tmp_path_factory.mktemp("""data""") / """dataset_with_unicode_new_lines.txt""")
			with open(_lowercase						,       """w"""						,       encoding="""utf-8""") as f:
						f.write(_lowercase)
			return path

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  )							->	Optional[int]:







			"""simple docstring"""
			return os.path.join("""tests"""						,       """features"""						,       """data"""						,       """test_image_rgb.jpg""")

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  )							->	List[Any]:







			"""simple docstring"""
			return os.path.join("""tests"""						,       """features"""						,       """data"""						,       """test_audio_44100.wav""")

@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[Any]						,       _lowercase    :       Any)							->	Union[str, Any]:







			"""simple docstring"""
			a__       :						str							=  tmp_path_factory.mktemp("""data""") / """dataset.img.zip"""
			with zipfile.ZipFile(_lowercase						,       """w""") as f:
						f.write(_lowercase						,       arcname=os.path.basename(_lowercase))
						f.write(_lowercase						,       arcname=os.path.basename(_lowercase).replace(""".jpg"""						,       """2.jpg"""))
			return path





@pytest.fixture(scope="""session""")
def      lowerCAmelCase_					(  _lowercase    :       Optional[Any])							->	Optional[int]:







			"""simple docstring"""
			a__       :						Union[str, Any]							=  tmp_path_factory.mktemp("""data_dir""")

			(data_dir / "subdir").mkdir()
			with open(data_dir / """subdir""" / """train.txt"""						,       """w""") as f:
						f.write("""foo\n""" * 10)
			with open(data_dir / """subdir""" / """test.txt"""						,       """w""") as f:
						f.write("""bar\n""" * 10)
			# hidden file
			with open(data_dir / """subdir""" / """.test.txt"""						,       """w""") as f:
						f.write("""bar\n""" * 10)

			# hidden directory
			(data_dir / ".subdir").mkdir()
			with open(data_dir / """.subdir""" / """train.txt"""						,       """w""") as f:
						f.write("""foo\n""" * 10)
			with open(data_dir / """.subdir""" / """test.txt"""						,       """w""") as f:
						f.write("""bar\n""" * 10)

			return data_dir









import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
_lowercase		:				Union[str, Any]      		=data_utils.TransfoXLTokenizer
_lowercase		:				Tuple      		=data_utils.TransfoXLCorpus
_lowercase		:				Optional[int]      		=data_utils
_lowercase		:				int      		=data_utils

def      lowerCAmelCase_					(  _lowercase    :       Optional[Any]						,       _lowercase    :       List[Any]						,       _lowercase    :       Optional[Any]						,       _lowercase    :       Optional[Any])							->	List[Any]:







			"""simple docstring"""
			if transfo_xl_dataset_file:
						# Convert a pre-processed corpus (see original TensorFlow repo)
						with open(_lowercase						,       """rb""") as fp:
									a__       :						Optional[Any]							=  pickle.load(_lowercase						,       encoding="""latin1""")
						# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
						a__       :						Dict							=  pytorch_dump_folder_path + """/""" + VOCAB_FILES_NAMES["""pretrained_vocab_file"""]
						print(F'''Save vocabulary to {pytorch_vocab_dump_path}''')
						a__       :						List[str]							=  corpus.vocab.__dict__
						torch.save(_lowercase						,       _lowercase)

						a__       :						Optional[int]							=  corpus.__dict__
						corpus_dict_no_vocab.pop("""vocab"""						,       _lowercase)
						a__       :						Tuple							=  pytorch_dump_folder_path + """/""" + CORPUS_NAME
						print(F'''Save dataset to {pytorch_dataset_dump_path}''')
						torch.save(_lowercase						,       _lowercase)

			if tf_checkpoint_path:
						# Convert a pre-trained TensorFlow model
						a__       :						Optional[Any]							=  os.path.abspath(_lowercase)
						a__       :						str							=  os.path.abspath(_lowercase)

						print(F'''Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.''')
						# Initialise PyTorch model
						if transfo_xl_config_file == "":
									a__       :						List[str]							=  TransfoXLConfig()
						else:
									a__       :						Any							=  TransfoXLConfig.from_json_file(_lowercase)
						print(F'''Building PyTorch model from configuration: {config}''')
						a__       :						Any							=  TransfoXLLMHeadModel(_lowercase)

						a__       :						Optional[Any]							=  load_tf_weights_in_transfo_xl(_lowercase						,       _lowercase						,       _lowercase)
						# Save pytorch-model
						a__       :						Tuple							=  os.path.join(_lowercase						,       _lowercase)
						a__       :						Optional[int]							=  os.path.join(_lowercase						,       _lowercase)
						print(F'''Save PyTorch model to {os.path.abspath(_lowercase)}''')
						torch.save(model.state_dict()						,       _lowercase)
						print(F'''Save configuration file to {os.path.abspath(_lowercase)}''')
						with open(_lowercase						,       """w"""						,       encoding="""utf-8""") as f:
									f.write(config.to_json_string())


if __name__ == "__main__":
		_lowercase		:				Tuple      		=argparse.ArgumentParser()
		parser.add_argument(
		    "--pytorch_dump_folder_path",
		    default=None,
		    type=str,
		    required=True,
		    help="Path to the folder to store the PyTorch model or dataset/vocab.",
		)
		parser.add_argument(
		    "--tf_checkpoint_path",
		    default="",
		    type=str,
		    help="An optional path to a TensorFlow checkpoint path to be converted.",
		)
		parser.add_argument(
		    "--transfo_xl_config_file",
		    default="",
		    type=str,
		    help=(
		        "An optional config json file corresponding to the pre-trained BERT model. \n"
		        "This specifies the model architecture."
		    ),
		)
		parser.add_argument(
		    "--transfo_xl_dataset_file",
		    default="",
		    type=str,
		    help="An optional dataset file to be converted in a vocabulary.",
		)
		_lowercase		:				Dict      		=parser.parse_args()
		convert_transfo_xl_checkpoint_to_pytorch(
		    args.tf_checkpoint_path,
		    args.transfo_xl_config_file,
		    args.pytorch_dump_folder_path,
		    args.transfo_xl_dataset_file,
		)




