




import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config





@pytest.fixture(scope="""session"""       )
def 						__A						(      ) ->			List[Any]:
	a									=				10
	a									=				datasets.Features(
	    {
	        """tokens""": datasets.Sequence(datasets.Value("""string"""       )       ),
	        """labels""": datasets.Sequence(datasets.ClassLabel(names=["""negative""", """positive"""]       )       ),
	        """answers""": datasets.Sequence(
	            {
	                """text""": datasets.Value("""string"""       ),
	                """answer_start""": datasets.Value("""int32"""       ),
	            }       ),
	        """id""": datasets.Value("""int64"""       ),
	    }       )
	a									=				datasets.Dataset.from_dict(
	    {
	        """tokens""": [["""foo"""] * 5] * n,
	        """labels""": [[1] * 5] * n,
	        """answers""": [{"""answer_start""": [97], """text""": ["""1976"""]}] * 10,
	        """id""": list(range(__snake_case       )       ),
	    }				,     features=__snake_case				,     )
	return dataset





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase       ) ->			Optional[Any]:
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """file.arrow"""       )
	dataset.map(cache_file_name=__snake_case       )
	return filename


# FILE_CONTENT + files


__UpperCamelCase   :			Optional[int]          = """\
    Text data.
    Second line of data."""





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			List[Any]:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """file.txt"""
	a									=				FILE_CONTENT
	with open(__snake_case				,     """w"""       ) as f:
		f.write(__snake_case       )
	return filename





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Optional[Any]:
	import bza

	a									=				tmp_path_factory.mktemp("""data"""       ) / """file.txt.bz2"""
	a									=				bytes(__snake_case				,     """utf-8"""       )
	with bza.open(__snake_case				,     """wb"""       ) as f:
		f.write(__snake_case       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			List[Any]:
	import gzip

	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """file.txt.gz"""       )
	a									=				bytes(__snake_case				,     """utf-8"""       )
	with gzip.open(__snake_case				,     """wb"""       ) as f:
		f.write(__snake_case       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Optional[int]:





	if datasets.config.LZ4_AVAILABLE:
		import lza.frame

		a									=				tmp_path_factory.mktemp("""data"""       ) / """file.txt.lz4"""
		a									=				bytes(__snake_case				,     """utf-8"""       )
		with lza.frame.open(__snake_case				,     """wb"""       ) as f:
			f.write(__snake_case       )
		return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase       ) ->			List[str]:





	if datasets.config.PY7ZR_AVAILABLE:
		import pyazr

		a									=				tmp_path_factory.mktemp("""data"""       ) / """file.txt.7z"""
		with pyazr.SevenZipFile(__snake_case				,     """w"""       ) as archive:
			archive.write(__snake_case				,     arcname=os.path.basename(__snake_case       )       )
		return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase       ) ->			int:
	import tarfile

	a									=				tmp_path_factory.mktemp("""data"""       ) / """file.txt.tar"""
	with tarfile.TarFile(__snake_case				,     """w"""       ) as f:
		f.add(__snake_case				,     arcname=os.path.basename(__snake_case       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Tuple:
	import lzma

	a									=				tmp_path_factory.mktemp("""data"""       ) / """file.txt.xz"""
	a									=				bytes(__snake_case				,     """utf-8"""       )
	with lzma.open(__snake_case				,     """wb"""       ) as f:
		f.write(__snake_case       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase       ) ->			List[str]:
	import zipfile

	a									=				tmp_path_factory.mktemp("""data"""       ) / """file.txt.zip"""
	with zipfile.ZipFile(__snake_case				,     """w"""       ) as f:
		f.write(__snake_case				,     arcname=os.path.basename(__snake_case       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			List[str]:





	if datasets.config.ZSTANDARD_AVAILABLE:
		import zstandard as zstd

		a									=				tmp_path_factory.mktemp("""data"""       ) / """file.txt.zst"""
		a									=				bytes(__snake_case				,     """utf-8"""       )
		with zstd.open(__snake_case				,     """wb"""       ) as f:
			f.write(__snake_case       )
		return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Optional[int]:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """file.xml"""
	a									=				textwrap.dedent(
	    """\\n    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n    <tmx version=\"1.4\">\n      <header segtype=\"sentence\" srclang=\"ca\" />\n      <body>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>"""       )
	with open(__snake_case				,     """w"""       ) as f:
		f.write(__snake_case       )
	return filename


__UpperCamelCase   :			List[str]          = [
    {"""col_1""": """0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """3""", """col_2""": 3, """col_3""": 3.0},
]
__UpperCamelCase   :			Dict          = [
    {"""col_1""": """4""", """col_2""": 4, """col_3""": 4.0},
    {"""col_1""": """5""", """col_2""": 5, """col_3""": 5.0},
]
__UpperCamelCase   :			Optional[Any]          = {
    """col_1""": ["""0""", """1""", """2""", """3"""],
    """col_2""": [0, 1, 2, 3],
    """col_3""": [0.0, 1.0, 2.0, 3.0],
}

__UpperCamelCase   :			List[Any]          = [
    {"""col_3""": 0.0, """col_1""": """0""", """col_2""": 0},
    {"""col_3""": 1.0, """col_1""": """1""", """col_2""": 1},
]

__UpperCamelCase   :			Union[str, Any]          = [
    {"""col_1""": """s0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """s1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """s2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """s3""", """col_2""": 3, """col_3""": 3.0},
]





@pytest.fixture(scope="""session"""       )
def 						__A						(      ) ->			Dict:
	return DATA_DICT_OF_LISTS





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			str:
	a									=				datasets.Dataset.from_dict(__snake_case       )
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset.arrow"""       )
	dataset.map(cache_file_name=__snake_case       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			str:
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset.sqlite"""       )
	with contextlib.closing(sqlitea.connect(__snake_case       )       ) as con:
		a									=				con.cursor()
		cur.execute("""CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)"""       )
		for item in DATA:
			cur.execute("""INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)"""				,     tuple(item.values()       )       )
		con.commit()
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Any:
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset.csv"""       )
	with open(__snake_case				,     """w"""				,     newline=""""""       ) as f:
		a									=				csv.DictWriter(__snake_case				,     fieldnames=["""col_1""", """col_2""", """col_3"""]       )
		writer.writeheader()
		for item in DATA:
			writer.writerow(__snake_case       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			int:
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset2.csv"""       )
	with open(__snake_case				,     """w"""				,     newline=""""""       ) as f:
		a									=				csv.DictWriter(__snake_case				,     fieldnames=["""col_1""", """col_2""", """col_3"""]       )
		writer.writeheader()
		for item in DATA:
			writer.writerow(__snake_case       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase       ) ->			Union[str, Any]:
	import bza

	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset.csv.bz2"""
	with open(__snake_case				,     """rb"""       ) as f:
		a									=				f.read()
	# data = bytes(FILE_CONTENT, "utf-8")
	with bza.open(__snake_case				,     """wb"""       ) as f:
		f.write(__snake_case       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase       ) ->			str:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset.csv.zip"""
	with zipfile.ZipFile(__snake_case				,     """w"""       ) as f:
		f.write(__snake_case				,     arcname=os.path.basename(__snake_case       )       )
		f.write(__snake_case				,     arcname=os.path.basename(__snake_case       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase       ) ->			List[Any]:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset.csv.zip"""
	with zipfile.ZipFile(__snake_case				,     """w"""       ) as f:
		f.write(__snake_case				,     arcname=os.path.basename(csv_path.replace(""".csv"""				,     """.CSV"""       )       )       )
		f.write(__snake_case				,     arcname=os.path.basename(csva_path.replace(""".csv"""				,     """.CSV"""       )       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase       ) ->			int:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset_with_dir.csv.zip"""
	with zipfile.ZipFile(__snake_case				,     """w"""       ) as f:
		f.write(__snake_case				,     arcname=os.path.join("""main_dir"""				,     os.path.basename(__snake_case       )       )       )
		f.write(__snake_case				,     arcname=os.path.join("""main_dir"""				,     os.path.basename(__snake_case       )       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Optional[Any]:
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset.parquet"""       )
	a									=				pa.schema(
	    {
	        """col_1""": pa.string(),
	        """col_2""": pa.intaa(),
	        """col_3""": pa.floataa(),
	    }       )
	with open(__snake_case				,     """wb"""       ) as f:
		a									=				pq.ParquetWriter(__snake_case				,     schema=__snake_case       )
		a									=				pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(__snake_case       )       )] for k in DATA[0]}				,     schema=__snake_case       )
		writer.write_table(__snake_case       )
		writer.close()
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Optional[Any]:
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset.json"""       )
	a									=				{"""data""": DATA}
	with open(__snake_case				,     """w"""       ) as f:
		json.dump(__snake_case				,     __snake_case       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Optional[Any]:
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset.json"""       )
	a									=				{"""data""": DATA_DICT_OF_LISTS}
	with open(__snake_case				,     """w"""       ) as f:
		json.dump(__snake_case				,     __snake_case       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Optional[Any]:
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset.jsonl"""       )
	with open(__snake_case				,     """w"""       ) as f:
		for item in DATA:
			f.write(json.dumps(__snake_case       ) + """\n"""       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			List[str]:
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset2.jsonl"""       )
	with open(__snake_case				,     """w"""       ) as f:
		for item in DATA:
			f.write(json.dumps(__snake_case       ) + """\n"""       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			str:
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset_312.jsonl"""       )
	with open(__snake_case				,     """w"""       ) as f:
		for item in DATA_312:
			f.write(json.dumps(__snake_case       ) + """\n"""       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Any:
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset-str.jsonl"""       )
	with open(__snake_case				,     """w"""       ) as f:
		for item in DATA_STR:
			f.write(json.dumps(__snake_case       ) + """\n"""       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase       ) ->			Union[str, Any]:
	import gzip

	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset.txt.gz"""       )
	with open(__snake_case				,     """rb"""       ) as orig_file:
		with gzip.open(__snake_case				,     """wb"""       ) as zipped_file:
			zipped_file.writelines(__snake_case       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase       ) ->			Tuple:
	import gzip

	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset.jsonl.gz"""       )
	with open(__snake_case				,     """rb"""       ) as orig_file:
		with gzip.open(__snake_case				,     """wb"""       ) as zipped_file:
			zipped_file.writelines(__snake_case       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase       ) ->			Tuple:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset.jsonl.zip"""
	with zipfile.ZipFile(__snake_case				,     """w"""       ) as f:
		f.write(__snake_case				,     arcname=os.path.basename(__snake_case       )       )
		f.write(__snake_case				,     arcname=os.path.basename(__snake_case       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase       ) ->			str:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset_nested.jsonl.zip"""
	with zipfile.ZipFile(__snake_case				,     """w"""       ) as f:
		f.write(__snake_case				,     arcname=os.path.join("""nested"""				,     os.path.basename(__snake_case       )       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase       ) ->			Any:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset_with_dir.jsonl.zip"""
	with zipfile.ZipFile(__snake_case				,     """w"""       ) as f:
		f.write(__snake_case				,     arcname=os.path.join("""main_dir"""				,     os.path.basename(__snake_case       )       )       )
		f.write(__snake_case				,     arcname=os.path.join("""main_dir"""				,     os.path.basename(__snake_case       )       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase       ) ->			Any:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset.jsonl.tar"""
	with tarfile.TarFile(__snake_case				,     """w"""       ) as f:
		f.add(__snake_case				,     arcname=os.path.basename(__snake_case       )       )
		f.add(__snake_case				,     arcname=os.path.basename(__snake_case       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase       ) ->			Union[str, Any]:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset_nested.jsonl.tar"""
	with tarfile.TarFile(__snake_case				,     """w"""       ) as f:
		f.add(__snake_case				,     arcname=os.path.join("""nested"""				,     os.path.basename(__snake_case       )       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Tuple:
	a									=				["""0""", """1""", """2""", """3"""]
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset.txt"""       )
	with open(__snake_case				,     """w"""       ) as f:
		for item in data:
			f.write(item + """\n"""       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Optional[int]:
	a									=				["""0""", """1""", """2""", """3"""]
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset2.txt"""       )
	with open(__snake_case				,     """w"""       ) as f:
		for item in data:
			f.write(item + """\n"""       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Dict:
	a									=				["""0""", """1""", """2""", """3"""]
	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset.abc"""
	with open(__snake_case				,     """w"""       ) as f:
		for item in data:
			f.write(item + """\n"""       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase       ) ->			Dict:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset.text.zip"""
	with zipfile.ZipFile(__snake_case				,     """w"""       ) as f:
		f.write(__snake_case				,     arcname=os.path.basename(__snake_case       )       )
		f.write(__snake_case				,     arcname=os.path.basename(__snake_case       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase       ) ->			str:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset_with_dir.text.zip"""
	with zipfile.ZipFile(__snake_case				,     """w"""       ) as f:
		f.write(__snake_case				,     arcname=os.path.join("""main_dir"""				,     os.path.basename(__snake_case       )       )       )
		f.write(__snake_case				,     arcname=os.path.join("""main_dir"""				,     os.path.basename(__snake_case       )       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase       ) ->			List[Any]:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset.ext.zip"""
	with zipfile.ZipFile(__snake_case				,     """w"""       ) as f:
		f.write(__snake_case				,     arcname=os.path.basename("""unsupported.ext"""       )       )
		f.write(__snake_case				,     arcname=os.path.basename("""unsupported_2.ext"""       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			int:
	a									=				"""\n""".join(["""First""", """Second\u2029with Unicode new line""", """Third"""]       )
	a									=				str(tmp_path_factory.mktemp("""data"""       ) / """dataset_with_unicode_new_lines.txt"""       )
	with open(__snake_case				,     """w"""				,     encoding="""utf-8"""       ) as f:
		f.write(__snake_case       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      ) ->			Optional[int]:
	return os.path.join("""tests"""				,     """features"""				,     """data"""				,     """test_image_rgb.jpg"""       )





@pytest.fixture(scope="""session"""       )
def 						__A						(      ) ->			Dict:
	return os.path.join("""tests"""				,     """features"""				,     """data"""				,     """test_audio_44100.wav"""       )





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase				,     __lowerCamelCase       ) ->			Tuple:
	a									=				tmp_path_factory.mktemp("""data"""       ) / """dataset.img.zip"""
	with zipfile.ZipFile(__snake_case				,     """w"""       ) as f:
		f.write(__snake_case				,     arcname=os.path.basename(__snake_case       )       )
		f.write(__snake_case				,     arcname=os.path.basename(__snake_case       ).replace(""".jpg"""				,     """2.jpg"""       )       )
	return path





@pytest.fixture(scope="""session"""       )
def 						__A						(      __lowerCamelCase       ) ->			Optional[int]:
	a									=				tmp_path_factory.mktemp("""data_dir"""       )

	(data_dir / "subdir").mkdir()
	with open(data_dir / """subdir""" / """train.txt"""				,     """w"""       ) as f:
		f.write("""foo\n""" * 10       )
	with open(data_dir / """subdir""" / """test.txt"""				,     """w"""       ) as f:
		f.write("""bar\n""" * 10       )
	# hidden file
	with open(data_dir / """subdir""" / """.test.txt"""				,     """w"""       ) as f:
		f.write("""bar\n""" * 10       )

	# hidden directory
	(data_dir / ".subdir").mkdir()
	with open(data_dir / """.subdir""" / """train.txt"""				,     """w"""       ) as f:
		f.write("""foo\n""" * 10       )
	with open(data_dir / """.subdir""" / """test.txt"""				,     """w"""       ) as f:
		f.write("""bar\n""" * 10       )

	return data_dir









"""simple docstring"""
import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import evaluate
import numpy as np
import torch
from datasets import load_dataset
from PIL import Image
from torchvision.transforms import (
    CenterCrop,
    Compose,
    Normalize,
    RandomHorizontalFlip,
    RandomResizedCrop,
    Resize,
    ToTensor,
)

import transformers
from transformers import (
    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
    AutoConfig,
    AutoImageProcessor,
    AutoModelForImageClassification,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import get_last_checkpoint
from transformers.utils import check_min_version, send_example_telemetry
from transformers.utils.versions import require_version
snake_case : Tuple     		=		logging.getLogger(__name__)

# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
check_min_version("""4.31.0""")

require_version("""datasets>=1.8.0""", """To fix: pip install -r examples/pytorch/image-classification/requirements.txt""")

snake_case : List[str]     		=		list(MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING.keys())
snake_case : int     		=		tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)


def 							A						(				__snake_case: str  )    ->		Tuple:

       """simple docstring"""





       with open(__snake_case ,     'rb'  ) as f:
              __magic_name__      				=    Image.open(__snake_case  )
              return im.convert('RGB'  )



@dataclass
class        UpperCamelCase__ :







    """simple docstring"""
    __UpperCAmelCase           = field(
        default=a_							,	metadata={
            """help""": """Name of a dataset from the hub (could be your own, possibly private dataset hosted on the hub)."""
        }							,	)
    __UpperCAmelCase           = field(
        default=a_							,	metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""})
    __UpperCAmelCase           = field(default=a_							,	metadata={"""help""": """A folder containing the training data."""})
    __UpperCAmelCase           = field(default=a_							,	metadata={"""help""": """A folder containing the validation data."""})
    __UpperCAmelCase           = field(
        default=0.15							,	metadata={"""help""": """Percent to split off of train for validation."""})
    __UpperCAmelCase           = field(
        default=a_							,	metadata={
            """help""": (
                """For debugging purposes or quicker training, truncate the number of training examples to this """
                """value if set."""
            )
        }							,	)
    __UpperCAmelCase           = field(
        default=a_							,	metadata={
            """help""": (
                """For debugging purposes or quicker training, truncate the number of evaluation examples to this """
                """value if set."""
            )
        }							,	)







    def 				a__			(		self :	Tuple ):





           '''simple docstring'''

           if self.dataset_name is None and (self.train_dir is None and self.validation_dir is None):
                  raise ValueError(
                      'You must specify either a dataset name from the hub or a train and/or validation directory.' )



@dataclass
class        UpperCamelCase__ :







    """simple docstring"""
    __UpperCAmelCase           = field(
        default="""google/vit-base-patch16-224-in21k"""							,	metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}							,	)
    __UpperCAmelCase           = field(
        default=a_							,	metadata={"""help""": """If training from scratch, pass a model type from the list: """ + """, """.join(a_)}							,	)
    __UpperCAmelCase           = field(
        default=a_							,	metadata={"""help""": """Pretrained config name or path if not the same as model_name"""})
    __UpperCAmelCase           = field(
        default=a_							,	metadata={"""help""": """Where do you want to store the pretrained models downloaded from s3"""})
    __UpperCAmelCase           = field(
        default="""main"""							,	metadata={"""help""": """The specific model version to use (can be a branch name, tag name or commit id)."""}							,	)
    __UpperCAmelCase           = field(default=a_							,	metadata={"""help""": """Name or path of preprocessor config."""})
    __UpperCAmelCase           = field(
        default=a_							,	metadata={
            """help""": (
                """Will use the token generated when running `huggingface-cli login` (necessary to use this script """
                """with private models)."""
            )
        }							,	)
    __UpperCAmelCase           = field(
        default=a_							,	metadata={"""help""": """Will enable to load a pretrained model whose head dimensions are different."""}							,	)


def 							A						(				__snake_case: Dict  )    ->		Any:

       """simple docstring"""





       __magic_name__      				=    torch.stack([example['pixel_values'] for example in examples]  )
       __magic_name__      				=    torch.tensor([example['labels'] for example in examples]  )
       return {"pixel_values": pixel_values, "labels": labels}


def 							A						(				)    ->		List[str]:

       """simple docstring"""





       __magic_name__      				=    HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)  )
       if len(sys.argv  ) == 2 and sys.argv[1].endswith('.json'  ):
              # If we pass only one argument to the script and it's the path to a json file,
              # let's parse it to get our arguments.
              __magic_name__				,  __magic_name__				,  __magic_name__      				=    parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]  )  )
       else:
              __magic_name__				,  __magic_name__				,  __magic_name__      				=    parser.parse_args_into_dataclasses()

       # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
       # information sent is the one passed as arguments along with your Python/PyTorch versions.
       send_example_telemetry('run_image_classification' ,     __snake_case ,     __snake_case  )

       # Setup logging
       logging.basicConfig(
           format='%(asctime)s - %(levelname)s - %(name)s - %(message)s' ,     datefmt='%m/%d/%Y %H:%M:%S' ,     handlers=[logging.StreamHandler(sys.stdout  )] ,     )

       if training_args.should_log:
              # The default of training_args.log_level is passive, so we set log level at info here to have that default.
              transformers.utils.logging.set_verbosity_info()

       __magic_name__      				=    training_args.get_process_log_level()
       logger.setLevel(__snake_case  )
       transformers.utils.logging.set_verbosity(__snake_case  )
       transformers.utils.logging.enable_default_handler()
       transformers.utils.logging.enable_explicit_format()

       # Log on each process the small summary:
       logger.warning(
           F"""Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"""
           + F"""distributed training: {bool(training_args.local_rank != -1  )}, 16-bits training: {training_args.fpaa}"""  )
       logger.info(F"""Training/evaluation parameters {training_args}"""  )

       # Detecting last checkpoint.
       __magic_name__      				=    None
       if os.path.isdir(training_args.output_dir  ) and training_args.do_train and not training_args.overwrite_output_dir:
              __magic_name__      				=    get_last_checkpoint(training_args.output_dir  )
              if last_checkpoint is None and len(os.listdir(training_args.output_dir  )  ) > 0:
                     raise ValueError(
                         F"""Output directory ({training_args.output_dir}) already exists and is not empty. """
                         'Use --overwrite_output_dir to overcome.'  )
              elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
                     logger.info(
                         F"""Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change """
                         'the `--output_dir` or add `--overwrite_output_dir` to train from scratch.'  )

    # Set seed before initializing model.
       set_seed(training_args.seed  )

       # Initialize our dataset and prepare it for the 'image-classification' task.
       if data_args.dataset_name is not None:
              __magic_name__      				=    load_dataset(
                  data_args.dataset_name ,     data_args.dataset_config_name ,     cache_dir=model_args.cache_dir ,     task='image-classification' ,     use_auth_token=True if model_args.use_auth_token else None ,     )
       else:
              __magic_name__      				=    {}
              if data_args.train_dir is not None:
                     __magic_name__      				=    os.path.join(data_args.train_dir ,     '**'  )
              if data_args.validation_dir is not None:
                     __magic_name__      				=    os.path.join(data_args.validation_dir ,     '**'  )
              __magic_name__      				=    load_dataset(
                  'imagefolder' ,     data_files=__snake_case ,     cache_dir=model_args.cache_dir ,     task='image-classification' ,     )

       # If we don't have a validation split, split off a percentage of train as validation.
       __magic_name__      				=    None if 'validation' in dataset.keys() else data_args.train_val_split
       if isinstance(data_args.train_val_split ,     __snake_case  ) and data_args.train_val_split > 0.0:
              __magic_name__      				=    dataset['train'].train_test_split(data_args.train_val_split  )
              __magic_name__      				=    split['train']
              __magic_name__      				=    split['test']

       # Prepare label mappings.
       # We'll include these in the model's config to get human readable labels in the Inference API.
       __magic_name__      				=    dataset['train'].features['labels'].names
       __magic_name__				,  __magic_name__      				=    {}, {}
       for i, label in enumerate(__snake_case  ):
              __magic_name__      				=    str(__snake_case  )
              __magic_name__      				=    label

       # Load the accuracy metric from the datasets package
       __magic_name__      				=    evaluate.load('accuracy'  )

       # Define our compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a
       # predictions and label_ids field) and has to return a dictionary string to float.
       def compute_metrics(__snake_case: str  ):
              return metric.compute(predictions=np.argmax(p.predictions ,     axis=1  ) ,     references=p.label_ids  )

       __magic_name__      				=    AutoConfig.from_pretrained(
           model_args.config_name or model_args.model_name_or_path ,     num_labels=len(__snake_case  ) ,     labelaid=__snake_case ,     idalabel=__snake_case ,     finetuning_task='image-classification' ,     cache_dir=model_args.cache_dir ,     revision=model_args.model_revision ,     use_auth_token=True if model_args.use_auth_token else None ,     )
       __magic_name__      				=    AutoModelForImageClassification.from_pretrained(
           model_args.model_name_or_path ,     from_tf=bool('.ckpt' in model_args.model_name_or_path  ) ,     config=__snake_case ,     cache_dir=model_args.cache_dir ,     revision=model_args.model_revision ,     use_auth_token=True if model_args.use_auth_token else None ,     ignore_mismatched_sizes=model_args.ignore_mismatched_sizes ,     )
       __magic_name__      				=    AutoImageProcessor.from_pretrained(
           model_args.image_processor_name or model_args.model_name_or_path ,     cache_dir=model_args.cache_dir ,     revision=model_args.model_revision ,     use_auth_token=True if model_args.use_auth_token else None ,     )

       # Define torchvision transforms to be applied to each image.
       if "shortest_edge" in image_processor.size:
              __magic_name__      				=    image_processor.size['shortest_edge']
       else:
              __magic_name__      				=    (image_processor.size['height'], image_processor.size['width'])
       __magic_name__      				=    Normalize(mean=image_processor.image_mean ,     std=image_processor.image_std  )
       __magic_name__      				=    Compose(
           [
               RandomResizedCrop(__snake_case  ),
               RandomHorizontalFlip(),
               ToTensor(),
               normalize,
           ]  )
       __magic_name__      				=    Compose(
           [
               Resize(__snake_case  ),
               CenterCrop(__snake_case  ),
               ToTensor(),
               normalize,
           ]  )

       def train_transforms(__snake_case: str  ):
              __magic_name__      				=    [
                  _train_transforms(pil_img.convert('RGB'  )  ) for pil_img in example_batch['image']
              ]
              return example_batch

       def val_transforms(__snake_case: List[Any]  ):
              __magic_name__      				=    [_val_transforms(pil_img.convert('RGB'  )  ) for pil_img in example_batch['image']]
              return example_batch

       if training_args.do_train:
              if "train" not in dataset:
                     raise ValueError('--do_train requires a train dataset'  )
              if data_args.max_train_samples is not None:
                     __magic_name__      				=    (
                         dataset['train'].shuffle(seed=training_args.seed  ).select(range(data_args.max_train_samples  )  )
                     )
              # Set the training transforms
              dataset["train"].set_transform(__snake_case  )

       if training_args.do_eval:
              if "validation" not in dataset:
                     raise ValueError('--do_eval requires a validation dataset'  )
              if data_args.max_eval_samples is not None:
                     __magic_name__      				=    (
                         dataset['validation'].shuffle(seed=training_args.seed  ).select(range(data_args.max_eval_samples  )  )
                     )
              # Set the validation transforms
              dataset["validation"].set_transform(__snake_case  )

       # Initalize our trainer
       __magic_name__      				=    Trainer(
           model=__snake_case ,     args=__snake_case ,     train_dataset=dataset['train'] if training_args.do_train else None ,     eval_dataset=dataset['validation'] if training_args.do_eval else None ,     compute_metrics=__snake_case ,     tokenizer=__snake_case ,     data_collator=__snake_case ,     )

       # Training
       if training_args.do_train:
              __magic_name__      				=    None
              if training_args.resume_from_checkpoint is not None:
                     __magic_name__      				=    training_args.resume_from_checkpoint
              elif last_checkpoint is not None:
                     __magic_name__      				=    last_checkpoint
              __magic_name__      				=    trainer.train(resume_from_checkpoint=__snake_case  )
              trainer.save_model()
              trainer.log_metrics('train' ,     train_result.metrics  )
              trainer.save_metrics('train' ,     train_result.metrics  )
              trainer.save_state()

       # Evaluation
       if training_args.do_eval:
              __magic_name__      				=    trainer.evaluate()
              trainer.log_metrics('eval' ,     __snake_case  )
              trainer.save_metrics('eval' ,     __snake_case  )

       # Write model card and (optionally) push to hub
       __magic_name__      				=    {
           'finetuned_from': model_args.model_name_or_path,
           'tasks': 'image-classification',
           'dataset': data_args.dataset_name,
           'tags': ['image-classification', 'vision'],
       }
       if training_args.push_to_hub:
              trainer.push_to_hub(**__snake_case  )
       else:
              trainer.create_model_card(**__snake_case  )


if __name__ == "__main__":
 main()