



import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config





@pytest.fixture(scope='session'   )
def 							A     (					)    ->			Dict:







			'''simple docstring'''
			_UpperCAmelCase			=       10
			_UpperCAmelCase			=       datasets.Features(
			    {
			        'tokens': datasets.Sequence(datasets.Value('string'   )   ),
			        'labels': datasets.Sequence(datasets.ClassLabel(names=['negative', 'positive']   )   ),
			        'answers': datasets.Sequence(
			            {
			                'text': datasets.Value('string'   ),
			                'answer_start': datasets.Value('int32'   ),
			            }   ),
			        'id': datasets.Value('int64'   ),
			    }   )
			_UpperCAmelCase			=       datasets.Dataset.from_dict(
			    {
			        'tokens': [['foo'] * 5] * n,
			        'labels': [[1] * 5] * n,
			        'answers': [{'answer_start': [97], 'text': ['1976']}] * 10,
			        'id': list(range(UpperCAmelCase_   )   ),
			    }							,       features=UpperCAmelCase_							,       )
			return dataset





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       str							,       _UpperCAmelCase    :       str   )    ->			int:







			'''simple docstring'''
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'file.arrow'   )
			dataset.map(cache_file_name=UpperCAmelCase_   )
			return filename


# FILE_CONTENT + files


UpperCAmelCase__       =	'\\n    Text data.\n    Second line of data.'





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Any   )    ->			Any:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'file.txt'
			_UpperCAmelCase			=       FILE_CONTENT
			with open(UpperCAmelCase_							,       'w'   ) as f:
						f.write(UpperCAmelCase_   )
			return filename





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Tuple   )    ->			List[str]:







			'''simple docstring'''
			import bza

			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'file.txt.bz2'
			_UpperCAmelCase			=       bytes(UpperCAmelCase_							,       'utf-8'   )
			with bza.open(UpperCAmelCase_							,       'wb'   ) as f:
						f.write(UpperCAmelCase_   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Any   )    ->			int:







			'''simple docstring'''
			import gzip

			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'file.txt.gz'   )
			_UpperCAmelCase			=       bytes(UpperCAmelCase_							,       'utf-8'   )
			with gzip.open(UpperCAmelCase_							,       'wb'   ) as f:
						f.write(UpperCAmelCase_   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       int   )    ->			List[Any]:







			'''simple docstring'''
			if datasets.config.LZ4_AVAILABLE:
						import lza.frame

						_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'file.txt.lz4'
						_UpperCAmelCase			=       bytes(UpperCAmelCase_							,       'utf-8'   )
						with lza.frame.open(UpperCAmelCase_							,       'wb'   ) as f:
									f.write(UpperCAmelCase_   )
						return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Optional[Any]							,       _UpperCAmelCase    :       List[Any]   )    ->			Dict:







			'''simple docstring'''
			if datasets.config.PY7ZR_AVAILABLE:
						import pyazr

						_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'file.txt.7z'
						with pyazr.SevenZipFile(UpperCAmelCase_							,       'w'   ) as archive:
									archive.write(UpperCAmelCase_							,       arcname=os.path.basename(UpperCAmelCase_   )   )
						return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Union[str, Any]							,       _UpperCAmelCase    :       int   )    ->			Optional[Any]:







			'''simple docstring'''
			import tarfile

			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'file.txt.tar'
			with tarfile.TarFile(UpperCAmelCase_							,       'w'   ) as f:
						f.add(UpperCAmelCase_							,       arcname=os.path.basename(UpperCAmelCase_   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       int   )    ->			Dict:







			'''simple docstring'''
			import lzma

			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'file.txt.xz'
			_UpperCAmelCase			=       bytes(UpperCAmelCase_							,       'utf-8'   )
			with lzma.open(UpperCAmelCase_							,       'wb'   ) as f:
						f.write(UpperCAmelCase_   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Tuple							,       _UpperCAmelCase    :       Any   )    ->			Union[str, Any]:







			'''simple docstring'''
			import zipfile

			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'file.txt.zip'
			with zipfile.ZipFile(UpperCAmelCase_							,       'w'   ) as f:
						f.write(UpperCAmelCase_							,       arcname=os.path.basename(UpperCAmelCase_   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Any   )    ->			Optional[int]:







			'''simple docstring'''
			if datasets.config.ZSTANDARD_AVAILABLE:
						import zstandard as zstd

						_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'file.txt.zst'
						_UpperCAmelCase			=       bytes(UpperCAmelCase_							,       'utf-8'   )
						with zstd.open(UpperCAmelCase_							,       'wb'   ) as f:
									f.write(UpperCAmelCase_   )
						return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       List[Any]   )    ->			Tuple:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'file.xml'
			_UpperCAmelCase			=       textwrap.dedent(
			    '\\n    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n    <tmx version=\"1.4\">\n      <header segtype=\"sentence\" srclang=\"ca\" />\n      <body>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>'   )
			with open(UpperCAmelCase_							,       'w'   ) as f:
						f.write(UpperCAmelCase_   )
			return filename


UpperCAmelCase__       =	[
    {'col_1': '0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': '1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': '2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': '3', 'col_2': 3, 'col_3': 3.0},
]
UpperCAmelCase__       =	[
    {'col_1': '4', 'col_2': 4, 'col_3': 4.0},
    {'col_1': '5', 'col_2': 5, 'col_3': 5.0},
]
UpperCAmelCase__       =	{
    'col_1': ['0', '1', '2', '3'],
    'col_2': [0, 1, 2, 3],
    'col_3': [0.0, 1.0, 2.0, 3.0],
}

UpperCAmelCase__       =	[
    {'col_3': 0.0, 'col_1': '0', 'col_2': 0},
    {'col_3': 1.0, 'col_1': '1', 'col_2': 1},
]

UpperCAmelCase__       =	[
    {'col_1': 's0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': 's1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': 's2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': 's3', 'col_2': 3, 'col_3': 3.0},
]





@pytest.fixture(scope='session'   )
def 							A     (					)    ->			Dict:







			'''simple docstring'''
			return DATA_DICT_OF_LISTS





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Tuple   )    ->			List[str]:







			'''simple docstring'''
			_UpperCAmelCase			=       datasets.Dataset.from_dict(UpperCAmelCase_   )
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset.arrow'   )
			dataset.map(cache_file_name=UpperCAmelCase_   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Dict   )    ->			List[str]:







			'''simple docstring'''
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset.sqlite'   )
			with contextlib.closing(sqlitea.connect(UpperCAmelCase_   )   ) as con:
						_UpperCAmelCase			=       con.cursor()
						cur.execute('CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'   )
						for item in DATA:
									cur.execute('INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'							,       tuple(item.values()   )   )
						con.commit()
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       int   )    ->			Optional[Any]:







			'''simple docstring'''
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset.csv'   )
			with open(UpperCAmelCase_							,       'w'							,       newline=''   ) as f:
						_UpperCAmelCase			=       csv.DictWriter(UpperCAmelCase_							,       fieldnames=['col_1', 'col_2', 'col_3']   )
						writer.writeheader()
						for item in DATA:
									writer.writerow(UpperCAmelCase_   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       List[str]   )    ->			List[Any]:







			'''simple docstring'''
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset2.csv'   )
			with open(UpperCAmelCase_							,       'w'							,       newline=''   ) as f:
						_UpperCAmelCase			=       csv.DictWriter(UpperCAmelCase_							,       fieldnames=['col_1', 'col_2', 'col_3']   )
						writer.writeheader()
						for item in DATA:
									writer.writerow(UpperCAmelCase_   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Dict							,       _UpperCAmelCase    :       Optional[int]   )    ->			List[Any]:







			'''simple docstring'''
			import bza

			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset.csv.bz2'
			with open(UpperCAmelCase_							,       'rb'   ) as f:
						_UpperCAmelCase			=       f.read()
			# data = bytes(FILE_CONTENT, "utf-8")
			with bza.open(UpperCAmelCase_							,       'wb'   ) as f:
						f.write(UpperCAmelCase_   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Tuple							,       _UpperCAmelCase    :       Any							,       _UpperCAmelCase    :       List[Any]   )    ->			str:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset.csv.zip'
			with zipfile.ZipFile(UpperCAmelCase_							,       'w'   ) as f:
						f.write(UpperCAmelCase_							,       arcname=os.path.basename(UpperCAmelCase_   )   )
						f.write(UpperCAmelCase_							,       arcname=os.path.basename(UpperCAmelCase_   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Tuple							,       _UpperCAmelCase    :       Optional[int]							,       _UpperCAmelCase    :       Dict   )    ->			int:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset.csv.zip'
			with zipfile.ZipFile(UpperCAmelCase_							,       'w'   ) as f:
						f.write(UpperCAmelCase_							,       arcname=os.path.basename(csv_path.replace('.csv'							,       '.CSV'   )   )   )
						f.write(UpperCAmelCase_							,       arcname=os.path.basename(csva_path.replace('.csv'							,       '.CSV'   )   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Any							,       _UpperCAmelCase    :       Any							,       _UpperCAmelCase    :       int   )    ->			List[Any]:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset_with_dir.csv.zip'
			with zipfile.ZipFile(UpperCAmelCase_							,       'w'   ) as f:
						f.write(UpperCAmelCase_							,       arcname=os.path.join('main_dir'							,       os.path.basename(UpperCAmelCase_   )   )   )
						f.write(UpperCAmelCase_							,       arcname=os.path.join('main_dir'							,       os.path.basename(UpperCAmelCase_   )   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       str   )    ->			Optional[Any]:







			'''simple docstring'''
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset.parquet'   )
			_UpperCAmelCase			=       pa.schema(
			    {
			        'col_1': pa.string(),
			        'col_2': pa.intaa(),
			        'col_3': pa.floataa(),
			    }   )
			with open(UpperCAmelCase_							,       'wb'   ) as f:
						_UpperCAmelCase			=       pq.ParquetWriter(UpperCAmelCase_							,       schema=UpperCAmelCase_   )
						_UpperCAmelCase			=       pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(UpperCAmelCase_   )   )] for k in DATA[0]}							,       schema=UpperCAmelCase_   )
						writer.write_table(UpperCAmelCase_   )
						writer.close()
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Union[str, Any]   )    ->			Any:







			'''simple docstring'''
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset.json'   )
			_UpperCAmelCase			=       {'data': DATA}
			with open(UpperCAmelCase_							,       'w'   ) as f:
						json.dump(UpperCAmelCase_							,       UpperCAmelCase_   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       str   )    ->			List[str]:







			'''simple docstring'''
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset.json'   )
			_UpperCAmelCase			=       {'data': DATA_DICT_OF_LISTS}
			with open(UpperCAmelCase_							,       'w'   ) as f:
						json.dump(UpperCAmelCase_							,       UpperCAmelCase_   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Any   )    ->			Tuple:







			'''simple docstring'''
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset.jsonl'   )
			with open(UpperCAmelCase_							,       'w'   ) as f:
						for item in DATA:
									f.write(json.dumps(UpperCAmelCase_   ) + '\n'   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Any   )    ->			List[str]:







			'''simple docstring'''
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset2.jsonl'   )
			with open(UpperCAmelCase_							,       'w'   ) as f:
						for item in DATA:
									f.write(json.dumps(UpperCAmelCase_   ) + '\n'   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Any   )    ->			int:







			'''simple docstring'''
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset_312.jsonl'   )
			with open(UpperCAmelCase_							,       'w'   ) as f:
						for item in DATA_312:
									f.write(json.dumps(UpperCAmelCase_   ) + '\n'   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Tuple   )    ->			int:







			'''simple docstring'''
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset-str.jsonl'   )
			with open(UpperCAmelCase_							,       'w'   ) as f:
						for item in DATA_STR:
									f.write(json.dumps(UpperCAmelCase_   ) + '\n'   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       List[str]							,       _UpperCAmelCase    :       Tuple   )    ->			Union[str, Any]:







			'''simple docstring'''
			import gzip

			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset.txt.gz'   )
			with open(UpperCAmelCase_							,       'rb'   ) as orig_file:
						with gzip.open(UpperCAmelCase_							,       'wb'   ) as zipped_file:
									zipped_file.writelines(UpperCAmelCase_   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       List[Any]							,       _UpperCAmelCase    :       int   )    ->			Optional[int]:







			'''simple docstring'''
			import gzip

			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset.jsonl.gz'   )
			with open(UpperCAmelCase_							,       'rb'   ) as orig_file:
						with gzip.open(UpperCAmelCase_							,       'wb'   ) as zipped_file:
									zipped_file.writelines(UpperCAmelCase_   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       List[str]							,       _UpperCAmelCase    :       List[str]							,       _UpperCAmelCase    :       Any   )    ->			Optional[int]:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset.jsonl.zip'
			with zipfile.ZipFile(UpperCAmelCase_							,       'w'   ) as f:
						f.write(UpperCAmelCase_							,       arcname=os.path.basename(UpperCAmelCase_   )   )
						f.write(UpperCAmelCase_							,       arcname=os.path.basename(UpperCAmelCase_   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Any							,       _UpperCAmelCase    :       int							,       _UpperCAmelCase    :       int							,       _UpperCAmelCase    :       List[str]   )    ->			Union[str, Any]:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset_nested.jsonl.zip'
			with zipfile.ZipFile(UpperCAmelCase_							,       'w'   ) as f:
						f.write(UpperCAmelCase_							,       arcname=os.path.join('nested'							,       os.path.basename(UpperCAmelCase_   )   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       str							,       _UpperCAmelCase    :       str							,       _UpperCAmelCase    :       Dict   )    ->			int:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset_with_dir.jsonl.zip'
			with zipfile.ZipFile(UpperCAmelCase_							,       'w'   ) as f:
						f.write(UpperCAmelCase_							,       arcname=os.path.join('main_dir'							,       os.path.basename(UpperCAmelCase_   )   )   )
						f.write(UpperCAmelCase_							,       arcname=os.path.join('main_dir'							,       os.path.basename(UpperCAmelCase_   )   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       List[str]							,       _UpperCAmelCase    :       Tuple							,       _UpperCAmelCase    :       List[Any]   )    ->			Union[str, Any]:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset.jsonl.tar'
			with tarfile.TarFile(UpperCAmelCase_							,       'w'   ) as f:
						f.add(UpperCAmelCase_							,       arcname=os.path.basename(UpperCAmelCase_   )   )
						f.add(UpperCAmelCase_							,       arcname=os.path.basename(UpperCAmelCase_   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Optional[Any]							,       _UpperCAmelCase    :       int							,       _UpperCAmelCase    :       Optional[Any]							,       _UpperCAmelCase    :       Optional[int]   )    ->			Optional[Any]:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset_nested.jsonl.tar'
			with tarfile.TarFile(UpperCAmelCase_							,       'w'   ) as f:
						f.add(UpperCAmelCase_							,       arcname=os.path.join('nested'							,       os.path.basename(UpperCAmelCase_   )   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       str   )    ->			Dict:







			'''simple docstring'''
			_UpperCAmelCase			=       ['0', '1', '2', '3']
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset.txt'   )
			with open(UpperCAmelCase_							,       'w'   ) as f:
						for item in data:
									f.write(item + '\n'   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       int   )    ->			Optional[Any]:







			'''simple docstring'''
			_UpperCAmelCase			=       ['0', '1', '2', '3']
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset2.txt'   )
			with open(UpperCAmelCase_							,       'w'   ) as f:
						for item in data:
									f.write(item + '\n'   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       int   )    ->			List[Any]:







			'''simple docstring'''
			_UpperCAmelCase			=       ['0', '1', '2', '3']
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset.abc'
			with open(UpperCAmelCase_							,       'w'   ) as f:
						for item in data:
									f.write(item + '\n'   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       str							,       _UpperCAmelCase    :       Tuple							,       _UpperCAmelCase    :       Union[str, Any]   )    ->			Optional[Any]:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset.text.zip'
			with zipfile.ZipFile(UpperCAmelCase_							,       'w'   ) as f:
						f.write(UpperCAmelCase_							,       arcname=os.path.basename(UpperCAmelCase_   )   )
						f.write(UpperCAmelCase_							,       arcname=os.path.basename(UpperCAmelCase_   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Union[str, Any]							,       _UpperCAmelCase    :       str							,       _UpperCAmelCase    :       List[Any]   )    ->			Dict:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset_with_dir.text.zip'
			with zipfile.ZipFile(UpperCAmelCase_							,       'w'   ) as f:
						f.write(UpperCAmelCase_							,       arcname=os.path.join('main_dir'							,       os.path.basename(UpperCAmelCase_   )   )   )
						f.write(UpperCAmelCase_							,       arcname=os.path.join('main_dir'							,       os.path.basename(UpperCAmelCase_   )   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Tuple							,       _UpperCAmelCase    :       Optional[Any]							,       _UpperCAmelCase    :       int   )    ->			Tuple:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset.ext.zip'
			with zipfile.ZipFile(UpperCAmelCase_							,       'w'   ) as f:
						f.write(UpperCAmelCase_							,       arcname=os.path.basename('unsupported.ext'   )   )
						f.write(UpperCAmelCase_							,       arcname=os.path.basename('unsupported_2.ext'   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       Optional[int]   )    ->			Optional[Any]:







			'''simple docstring'''
			_UpperCAmelCase			=       '\n'.join(['First', 'Second\u2029with Unicode new line', 'Third']   )
			_UpperCAmelCase			=       str(tmp_path_factory.mktemp('data'   ) / 'dataset_with_unicode_new_lines.txt'   )
			with open(UpperCAmelCase_							,       'w'							,       encoding='utf-8'   ) as f:
						f.write(UpperCAmelCase_   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					)    ->			int:







			'''simple docstring'''
			return os.path.join('tests'							,       'features'							,       'data'							,       'test_image_rgb.jpg'   )





@pytest.fixture(scope='session'   )
def 							A     (					)    ->			Union[str, Any]:







			'''simple docstring'''
			return os.path.join('tests'							,       'features'							,       'data'							,       'test_audio_44100.wav'   )





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       str							,       _UpperCAmelCase    :       Tuple   )    ->			Tuple:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data'   ) / 'dataset.img.zip'
			with zipfile.ZipFile(UpperCAmelCase_							,       'w'   ) as f:
						f.write(UpperCAmelCase_							,       arcname=os.path.basename(UpperCAmelCase_   )   )
						f.write(UpperCAmelCase_							,       arcname=os.path.basename(UpperCAmelCase_   ).replace('.jpg'							,       '2.jpg'   )   )
			return path





@pytest.fixture(scope='session'   )
def 							A     (					_UpperCAmelCase    :       List[Any]   )    ->			Dict:







			'''simple docstring'''
			_UpperCAmelCase			=       tmp_path_factory.mktemp('data_dir'   )

			(data_dir / "subdir").mkdir()
			with open(data_dir / 'subdir' / 'train.txt'							,       'w'   ) as f:
						f.write('foo\n' * 10   )
			with open(data_dir / 'subdir' / 'test.txt'							,       'w'   ) as f:
						f.write('bar\n' * 10   )
			# hidden file
			with open(data_dir / 'subdir' / '.test.txt'							,       'w'   ) as f:
						f.write('bar\n' * 10   )

			# hidden directory
			(data_dir / ".subdir").mkdir()
			with open(data_dir / '.subdir' / 'train.txt'							,       'w'   ) as f:
						f.write('foo\n' * 10   )
			with open(data_dir / '.subdir' / 'test.txt'							,       'w'   ) as f:
						f.write('bar\n' * 10   )

			return data_dir





import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class       __lowerCAmelCase   (      A    ,						unittest.TestCase						):
			UpperCamelCase        =					CLIPTokenizer
			UpperCamelCase        =					CLIPTokenizerFast
			UpperCamelCase        =					True
			UpperCamelCase        =					{}
			UpperCamelCase        =					False



			def    _lowerCamelCase	(						self       :  List[str])       ->   List[str]:



						"""simple docstring"""






						super().setUp()

						# fmt: off
						_UpperCAmelCase			=       ['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
						# fmt: on
						_UpperCAmelCase			=       dict(zip(A			,		range(len(A))))
						_UpperCAmelCase			=       ['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
						_UpperCAmelCase			=       {'unk_token': '<unk>'}

						_UpperCAmelCase			=       os.path.join(self.tmpdirname			,		VOCAB_FILES_NAMES['vocab_file'])
						_UpperCAmelCase			=       os.path.join(self.tmpdirname			,		VOCAB_FILES_NAMES['merges_file'])
						with open(self.vocab_file			,		'w'			,		encoding='utf-8') as fp:
									fp.write(json.dumps(A) + '\n')
						with open(self.merges_file			,		'w'			,		encoding='utf-8') as fp:
									fp.write('\n'.join(A))



			def    _lowerCamelCase	(						self       :  Optional[Any]			,		**A       :  str)       ->   Optional[int]:



						"""simple docstring"""






						kwargs.update(self.special_tokens_map)
						return CLIPTokenizer.from_pretrained(self.tmpdirname			,		**A)



			def    _lowerCamelCase	(						self       :  Any			,		**A       :  Dict)       ->   Union[str, Any]:



						"""simple docstring"""






						kwargs.update(self.special_tokens_map)
						return CLIPTokenizerFast.from_pretrained(self.tmpdirname			,		**A)



			def    _lowerCamelCase	(						self       :  Optional[int]			,		A       :  Union[str, Any])       ->   int:



						"""simple docstring"""






						_UpperCAmelCase			=       'lower newer'
						_UpperCAmelCase			=       'lower newer'
						return input_text, output_text



			def    _lowerCamelCase	(						self       :  Dict)       ->   Any:



						"""simple docstring"""






						_UpperCAmelCase			=       CLIPTokenizer(self.vocab_file			,		self.merges_file			,		**self.special_tokens_map)
						_UpperCAmelCase			=       'lower newer'
						_UpperCAmelCase			=       ['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
						_UpperCAmelCase			=       tokenizer.tokenize(A)
						self.assertListEqual(A			,		A)

						_UpperCAmelCase			=       tokens + [tokenizer.unk_token]
						_UpperCAmelCase			=       [10, 2, 16, 9, 3, 2, 16, 20]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A)			,		A)



			@require_ftfy
			def    _lowerCamelCase	(						self       :  List[Any])       ->   Dict:



						"""simple docstring"""






						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
												_UpperCAmelCase			=       self.tokenizer_class.from_pretrained(A			,		**A)
												_UpperCAmelCase			=       self.rust_tokenizer_class.from_pretrained(A			,		**A)

												_UpperCAmelCase			=       'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
												_UpperCAmelCase			=       tokenizer_s.tokenize(A)
												_UpperCAmelCase			=       tokenizer_r.tokenize(A)

												self.assertListEqual(A			,		A)

												# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
												# with Tilde) encoded in 2 different ways
												_UpperCAmelCase			=       'xa\u0303y' + ' ' + 'x\xe3y'
												_UpperCAmelCase			=       tokenizer_s.tokenize(A)
												_UpperCAmelCase			=       tokenizer_r.tokenize(A)

												self.assertListEqual(A			,		A)

												# Test that the tokenization is identical on unicode of space type
												_UpperCAmelCase			=       [
												    '\u0009',  # (horizontal tab, '\t')
												    '\u000B',  # (vertical tab)
												    '\u000C',  # (form feed)
												    '\u0020',  # (space, ' ')
												    '\u200E',  # (left-to-right mark):w
												    '\u200F',  # (right-to-left mark)
												]
												for unicode_seq in spaces_unicodes:
															_UpperCAmelCase			=       tokenizer_s.tokenize(A)
															_UpperCAmelCase			=       tokenizer_r.tokenize(A)

															self.assertListEqual(A			,		A)

												# Test that the tokenization is identical on unicode of line break type
												_UpperCAmelCase			=       [
												    '\u000A',  # (line feed, '\n')
												    '\r\n',  # (carriage return and line feed, '\r\n')
												    '\u000D',  # (carriage return, '\r')
												    '\r',  # (carriage return, '\r')
												    '\u000D',  # (carriage return, '\r')
												    '\u2028',  # (line separator)
												    '\u2029',  # (paragraph separator)
												    # "\u0085", # (next line)
												]

												# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
												# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
												# space (and thus into an empty list).

												for unicode_seq in line_break_unicodes:
															_UpperCAmelCase			=       tokenizer_s.tokenize(A)
															_UpperCAmelCase			=       tokenizer_r.tokenize(A)

															self.assertListEqual(A			,		A)



			def    _lowerCamelCase	(						self       :  str)       ->   Optional[Any]:



						"""simple docstring"""






						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
												_UpperCAmelCase			=       'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
												_UpperCAmelCase			=       F"{text_of_1_token} {text_of_1_token}"

												_UpperCAmelCase			=       self.rust_tokenizer_class.from_pretrained(
												    A			,		use_fast=A			,		)
												_UpperCAmelCase			=       tokenizer_r(A			,		return_offsets_mapping=A			,		add_special_tokens=A)
												self.assertEqual(encoding.offset_mapping[0]			,		(0, len(A)))
												self.assertEqual(
												    encoding.offset_mapping[1]			,		(len(A) + 1, len(A) + 1 + len(A))			,		)

												_UpperCAmelCase			=       F" {text}"

												_UpperCAmelCase			=       self.rust_tokenizer_class.from_pretrained(
												    A			,		use_fast=A			,		)
												_UpperCAmelCase			=       tokenizer_r(A			,		return_offsets_mapping=A			,		add_special_tokens=A)
												self.assertEqual(encoding.offset_mapping[0]			,		(1, 1 + len(A)))
												self.assertEqual(
												    encoding.offset_mapping[1]			,		(1 + len(A) + 1, 1 + len(A) + 1 + len(A))			,		)



			def    _lowerCamelCase	(						self       :  Tuple)       ->   str:



						"""simple docstring"""






						with self.assertRaises(A) as context:
									self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer')

						self.assertTrue(
						    context.exception.args[0].startswith(
						        'The `backend_tokenizer` provided does not match the expected format.'))



			@require_ftfy
			def    _lowerCamelCase	(						self       :  int)       ->   int:



						"""simple docstring"""






						super().test_tokenization_python_rust_equals()



			def    _lowerCamelCase	(						self       :  Union[str, Any])       ->   Any:



						"""simple docstring"""






						pass
