




import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( )     -> List[Any]:



       """simple docstring"""

       A__   				=					10
       A__   				=					datasets.Features(
           {
               '''tokens''': datasets.Sequence(datasets.Value('''string'''		)		),
               '''labels''': datasets.Sequence(datasets.ClassLabel(names=['''negative''', '''positive''']		)		),
               '''answers''': datasets.Sequence(
                   {
                       '''text''': datasets.Value('''string'''		),
                       '''answer_start''': datasets.Value('''int32'''		),
                   }		),
               '''id''': datasets.Value('''int64'''		),
           }		)
       A__   				=					datasets.Dataset.from_dict(
           {
               '''tokens''': [['''foo'''] * 5] * n,
               '''labels''': [[1] * 5] * n,
               '''answers''': [{'''answer_start''': [97], '''text''': ['''1976''']}] * 10,
               '''id''': list(range(lowercase_		)		),
           }				,       features=lowercase_				,       )
       return dataset







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_		)     -> Any:



       """simple docstring"""

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''file.arrow'''		)
       dataset.map(cache_file_name=lowercase_		)
       return filename


# FILE_CONTENT + files


_lowerCamelCase			:	List[Any]							 =     """\
    Text data.
    Second line of data."""







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Dict:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''file.txt'''
       A__   				=					FILE_CONTENT
       with open(lowercase_				,       '''w'''		) as f:
              f.write(lowercase_		)
       return filename







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> List[str]:



       """simple docstring"""

       import bza

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''file.txt.bz2'''
       A__   				=					bytes(lowercase_				,       '''utf-8'''		)
       with bza.open(lowercase_				,       '''wb'''		) as f:
              f.write(lowercase_		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Any:



       """simple docstring"""

       import gzip

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''file.txt.gz'''		)
       A__   				=					bytes(lowercase_				,       '''utf-8'''		)
       with gzip.open(lowercase_				,       '''wb'''		) as f:
              f.write(lowercase_		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Dict:



       """simple docstring"""

       if datasets.config.LZ4_AVAILABLE:
              import lza.frame

              A__   				=					tmp_path_factory.mktemp('''data'''		) / '''file.txt.lz4'''
              A__   				=					bytes(lowercase_				,       '''utf-8'''		)
              with lza.frame.open(lowercase_				,       '''wb'''		) as f:
                     f.write(lowercase_		)
              return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_		)     -> List[str]:



       """simple docstring"""

       if datasets.config.PY7ZR_AVAILABLE:
              import pyazr

              A__   				=					tmp_path_factory.mktemp('''data'''		) / '''file.txt.7z'''
              with pyazr.SevenZipFile(lowercase_				,       '''w'''		) as archive:
                     archive.write(lowercase_				,       arcname=os.path.basename(lowercase_		)		)
              return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_		)     -> Union[str, Any]:



       """simple docstring"""

       import tarfile

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''file.txt.tar'''
       with tarfile.TarFile(lowercase_				,       '''w'''		) as f:
              f.add(lowercase_				,       arcname=os.path.basename(lowercase_		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Any:



       """simple docstring"""

       import lzma

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''file.txt.xz'''
       A__   				=					bytes(lowercase_				,       '''utf-8'''		)
       with lzma.open(lowercase_				,       '''wb'''		) as f:
              f.write(lowercase_		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_		)     -> Dict:



       """simple docstring"""

       import zipfile

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''file.txt.zip'''
       with zipfile.ZipFile(lowercase_				,       '''w'''		) as f:
              f.write(lowercase_				,       arcname=os.path.basename(lowercase_		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> List[str]:



       """simple docstring"""

       if datasets.config.ZSTANDARD_AVAILABLE:
              import zstandard as zstd

              A__   				=					tmp_path_factory.mktemp('''data'''		) / '''file.txt.zst'''
              A__   				=					bytes(lowercase_				,       '''utf-8'''		)
              with zstd.open(lowercase_				,       '''wb'''		) as f:
                     f.write(lowercase_		)
              return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Union[str, Any]:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''file.xml'''
       A__   				=					textwrap.dedent(
           '''\
    <?xml version="1.0" encoding="UTF-8" ?>
    <tmx version="1.4">
      <header segtype="sentence" srclang="ca" />
      <body>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang="en"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang="en"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang="en"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang="en"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang="en"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>'''		)
       with open(lowercase_				,       '''w'''		) as f:
              f.write(lowercase_		)
       return filename


_lowerCamelCase			:	Union[str, Any]							 =     [
    {"""col_1""": """0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """3""", """col_2""": 3, """col_3""": 3.0},
]
_lowerCamelCase			:	Tuple							 =     [
    {"""col_1""": """4""", """col_2""": 4, """col_3""": 4.0},
    {"""col_1""": """5""", """col_2""": 5, """col_3""": 5.0},
]
_lowerCamelCase			:	str							 =     {
    """col_1""": ["""0""", """1""", """2""", """3"""],
    """col_2""": [0, 1, 2, 3],
    """col_3""": [0.0, 1.0, 2.0, 3.0],
}

_lowerCamelCase			:	List[Any]							 =     [
    {"""col_3""": 0.0, """col_1""": """0""", """col_2""": 0},
    {"""col_3""": 1.0, """col_1""": """1""", """col_2""": 1},
]

_lowerCamelCase			:	Dict							 =     [
    {"""col_1""": """s0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """s1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """s2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """s3""", """col_2""": 3, """col_3""": 3.0},
]







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( )     -> Union[str, Any]:



       """simple docstring"""

       return DATA_DICT_OF_LISTS







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Union[str, Any]:



       """simple docstring"""

       A__   				=					datasets.Dataset.from_dict(lowercase_		)
       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset.arrow'''		)
       dataset.map(cache_file_name=lowercase_		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Optional[int]:



       """simple docstring"""

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset.sqlite'''		)
       with contextlib.closing(sqlitea.connect(lowercase_		)		) as con:
              A__   				=					con.cursor()
              cur.execute('''CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'''		)
              for item in DATA:
                     cur.execute('''INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'''				,       tuple(item.values()		)		)
              con.commit()
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Optional[int]:



       """simple docstring"""

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset.csv'''		)
       with open(lowercase_				,       '''w'''				,       newline=''''''		) as f:
              A__   				=					csv.DictWriter(lowercase_				,       fieldnames=['''col_1''', '''col_2''', '''col_3''']		)
              writer.writeheader()
              for item in DATA:
                     writer.writerow(lowercase_		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> List[str]:



       """simple docstring"""

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset2.csv'''		)
       with open(lowercase_				,       '''w'''				,       newline=''''''		) as f:
              A__   				=					csv.DictWriter(lowercase_				,       fieldnames=['''col_1''', '''col_2''', '''col_3''']		)
              writer.writeheader()
              for item in DATA:
                     writer.writerow(lowercase_		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_		)     -> List[str]:



       """simple docstring"""

       import bza

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset.csv.bz2'''
       with open(lowercase_				,       '''rb'''		) as f:
              A__   				=					f.read()
       # data = bytes(FILE_CONTENT, "utf-8")
       with bza.open(lowercase_				,       '''wb'''		) as f:
              f.write(lowercase_		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_				,       lowercase_		)     -> Optional[int]:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset.csv.zip'''
       with zipfile.ZipFile(lowercase_				,       '''w'''		) as f:
              f.write(lowercase_				,       arcname=os.path.basename(lowercase_		)		)
              f.write(lowercase_				,       arcname=os.path.basename(lowercase_		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_				,       lowercase_		)     -> Optional[Any]:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset.csv.zip'''
       with zipfile.ZipFile(lowercase_				,       '''w'''		) as f:
              f.write(lowercase_				,       arcname=os.path.basename(csv_path.replace('''.csv'''				,       '''.CSV'''		)		)		)
              f.write(lowercase_				,       arcname=os.path.basename(csva_path.replace('''.csv'''				,       '''.CSV'''		)		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_				,       lowercase_		)     -> Any:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset_with_dir.csv.zip'''
       with zipfile.ZipFile(lowercase_				,       '''w'''		) as f:
              f.write(lowercase_				,       arcname=os.path.join('''main_dir'''				,       os.path.basename(lowercase_		)		)		)
              f.write(lowercase_				,       arcname=os.path.join('''main_dir'''				,       os.path.basename(lowercase_		)		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> List[Any]:



       """simple docstring"""

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset.parquet'''		)
       A__   				=					pa.schema(
           {
               '''col_1''': pa.string(),
               '''col_2''': pa.intaa(),
               '''col_3''': pa.floataa(),
           }		)
       with open(lowercase_				,       '''wb'''		) as f:
              A__   				=					pq.ParquetWriter(lowercase_				,       schema=lowercase_		)
              A__   				=					pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(lowercase_		)		)] for k in DATA[0]}				,       schema=lowercase_		)
              writer.write_table(lowercase_		)
              writer.close()
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Union[str, Any]:



       """simple docstring"""

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset.json'''		)
       A__   				=					{'''data''': DATA}
       with open(lowercase_				,       '''w'''		) as f:
              json.dump(lowercase_				,       lowercase_		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> List[Any]:



       """simple docstring"""

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset.json'''		)
       A__   				=					{'''data''': DATA_DICT_OF_LISTS}
       with open(lowercase_				,       '''w'''		) as f:
              json.dump(lowercase_				,       lowercase_		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> List[Any]:



       """simple docstring"""

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset.jsonl'''		)
       with open(lowercase_				,       '''w'''		) as f:
              for item in DATA:
                     f.write(json.dumps(lowercase_		) + '''\n'''		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Union[str, Any]:



       """simple docstring"""

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset2.jsonl'''		)
       with open(lowercase_				,       '''w'''		) as f:
              for item in DATA:
                     f.write(json.dumps(lowercase_		) + '''\n'''		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> List[Any]:



       """simple docstring"""

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset_312.jsonl'''		)
       with open(lowercase_				,       '''w'''		) as f:
              for item in DATA_312:
                     f.write(json.dumps(lowercase_		) + '''\n'''		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Optional[int]:



       """simple docstring"""

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset-str.jsonl'''		)
       with open(lowercase_				,       '''w'''		) as f:
              for item in DATA_STR:
                     f.write(json.dumps(lowercase_		) + '''\n'''		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_		)     -> int:



       """simple docstring"""

       import gzip

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset.txt.gz'''		)
       with open(lowercase_				,       '''rb'''		) as orig_file:
              with gzip.open(lowercase_				,       '''wb'''		) as zipped_file:
                     zipped_file.writelines(lowercase_		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_		)     -> Union[str, Any]:



       """simple docstring"""

       import gzip

       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset.jsonl.gz'''		)
       with open(lowercase_				,       '''rb'''		) as orig_file:
              with gzip.open(lowercase_				,       '''wb'''		) as zipped_file:
                     zipped_file.writelines(lowercase_		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_				,       lowercase_		)     -> int:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset.jsonl.zip'''
       with zipfile.ZipFile(lowercase_				,       '''w'''		) as f:
              f.write(lowercase_				,       arcname=os.path.basename(lowercase_		)		)
              f.write(lowercase_				,       arcname=os.path.basename(lowercase_		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_				,       lowercase_				,       lowercase_		)     -> Any:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset_nested.jsonl.zip'''
       with zipfile.ZipFile(lowercase_				,       '''w'''		) as f:
              f.write(lowercase_				,       arcname=os.path.join('''nested'''				,       os.path.basename(lowercase_		)		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_				,       lowercase_		)     -> Tuple:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset_with_dir.jsonl.zip'''
       with zipfile.ZipFile(lowercase_				,       '''w'''		) as f:
              f.write(lowercase_				,       arcname=os.path.join('''main_dir'''				,       os.path.basename(lowercase_		)		)		)
              f.write(lowercase_				,       arcname=os.path.join('''main_dir'''				,       os.path.basename(lowercase_		)		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_				,       lowercase_		)     -> Optional[Any]:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset.jsonl.tar'''
       with tarfile.TarFile(lowercase_				,       '''w'''		) as f:
              f.add(lowercase_				,       arcname=os.path.basename(lowercase_		)		)
              f.add(lowercase_				,       arcname=os.path.basename(lowercase_		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_				,       lowercase_				,       lowercase_		)     -> int:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset_nested.jsonl.tar'''
       with tarfile.TarFile(lowercase_				,       '''w'''		) as f:
              f.add(lowercase_				,       arcname=os.path.join('''nested'''				,       os.path.basename(lowercase_		)		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Tuple:



       """simple docstring"""

       A__   				=					['''0''', '''1''', '''2''', '''3''']
       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset.txt'''		)
       with open(lowercase_				,       '''w'''		) as f:
              for item in data:
                     f.write(item + '''\n'''		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> str:



       """simple docstring"""

       A__   				=					['''0''', '''1''', '''2''', '''3''']
       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset2.txt'''		)
       with open(lowercase_				,       '''w'''		) as f:
              for item in data:
                     f.write(item + '''\n'''		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Tuple:



       """simple docstring"""

       A__   				=					['''0''', '''1''', '''2''', '''3''']
       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset.abc'''
       with open(lowercase_				,       '''w'''		) as f:
              for item in data:
                     f.write(item + '''\n'''		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_				,       lowercase_		)     -> List[str]:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset.text.zip'''
       with zipfile.ZipFile(lowercase_				,       '''w'''		) as f:
              f.write(lowercase_				,       arcname=os.path.basename(lowercase_		)		)
              f.write(lowercase_				,       arcname=os.path.basename(lowercase_		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_				,       lowercase_		)     -> List[str]:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset_with_dir.text.zip'''
       with zipfile.ZipFile(lowercase_				,       '''w'''		) as f:
              f.write(lowercase_				,       arcname=os.path.join('''main_dir'''				,       os.path.basename(lowercase_		)		)		)
              f.write(lowercase_				,       arcname=os.path.join('''main_dir'''				,       os.path.basename(lowercase_		)		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_				,       lowercase_		)     -> str:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset.ext.zip'''
       with zipfile.ZipFile(lowercase_				,       '''w'''		) as f:
              f.write(lowercase_				,       arcname=os.path.basename('''unsupported.ext'''		)		)
              f.write(lowercase_				,       arcname=os.path.basename('''unsupported_2.ext'''		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Dict:



       """simple docstring"""

       A__   				=					'''\n'''.join(['''First''', '''Second\u2029with Unicode new line''', '''Third''']		)
       A__   				=					str(tmp_path_factory.mktemp('''data'''		) / '''dataset_with_unicode_new_lines.txt'''		)
       with open(lowercase_				,       '''w'''				,       encoding='''utf-8'''		) as f:
              f.write(lowercase_		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( )     -> str:



       """simple docstring"""

       return os.path.join('''tests'''				,       '''features'''				,       '''data'''				,       '''test_image_rgb.jpg'''		)







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( )     -> Dict:



       """simple docstring"""

       return os.path.join('''tests'''				,       '''features'''				,       '''data'''				,       '''test_audio_44100.wav'''		)







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_		)     -> Union[str, Any]:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data'''		) / '''dataset.img.zip'''
       with zipfile.ZipFile(lowercase_				,       '''w'''		) as f:
              f.write(lowercase_				,       arcname=os.path.basename(lowercase_		)		)
              f.write(lowercase_				,       arcname=os.path.basename(lowercase_		).replace('''.jpg'''				,       '''2.jpg'''		)		)
       return path







@pytest.fixture(scope='''session'''		)
def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> Dict:



       """simple docstring"""

       A__   				=					tmp_path_factory.mktemp('''data_dir'''		)

       (data_dir / "subdir").mkdir()
       with open(data_dir / '''subdir''' / '''train.txt'''				,       '''w'''		) as f:
              f.write('''foo\n''' * 10		)
       with open(data_dir / '''subdir''' / '''test.txt'''				,       '''w'''		) as f:
              f.write('''bar\n''' * 10		)
       # hidden file
       with open(data_dir / '''subdir''' / '''.test.txt'''				,       '''w'''		) as f:
              f.write('''bar\n''' * 10		)

       # hidden directory
       (data_dir / ".subdir").mkdir()
       with open(data_dir / '''.subdir''' / '''train.txt'''				,       '''w'''		) as f:
              f.write('''foo\n''' * 10		)
       with open(data_dir / '''.subdir''' / '''test.txt'''				,       '''w'''		) as f:
              f.write('''bar\n''' * 10		)

       return data_dir








import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, DDPMScheduler, StableDiffusionUpscalePipeline, UNetaDConditionModel
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()


class        UpperCamelCase_					(				unittest.TestCase  ):

    '''simple docstring'''






    def  SCREAMING_SNAKE_CASE							(    self   :			List[str])		->List[Any]:





           '''simple docstring'''

           super().tearDown()
           gc.collect()
           torch.cuda.empty_cache()


    @property
    def  SCREAMING_SNAKE_CASE							(    self   :			Union[str, Any])		->Dict:





           '''simple docstring'''

           A__   				=					1
           A__   				=					3
           A__   				=					(32, 32)

           A__   				=					floats_tensor((batch_size, num_channels) + sizes      ,				rng=random.Random(0)).to(UpperCAmelCase__)
           return image


    @property
    def  SCREAMING_SNAKE_CASE							(    self   :			Any)		->str:





           '''simple docstring'''

           torch.manual_seed(0)
           A__   				=					UNetaDConditionModel(
               block_out_channels=(32, 32, 64)      ,				layers_per_block=2      ,				sample_size=32      ,				in_channels=7      ,				out_channels=4      ,				down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''', '''CrossAttnDownBlock2D''')      ,				up_block_types=('''CrossAttnUpBlock2D''', '''CrossAttnUpBlock2D''', '''UpBlock2D''')      ,				cross_attention_dim=32      ,				attention_head_dim=8      ,				use_linear_projection=UpperCAmelCase__      ,				only_cross_attention=(True, True, False)      ,				num_class_embeds=100      ,				)
           return model


    @property
    def  SCREAMING_SNAKE_CASE							(    self   :			str)		->Dict:





           '''simple docstring'''

           torch.manual_seed(0)
           A__   				=					AutoencoderKL(
               block_out_channels=[32, 32, 64]      ,				in_channels=3      ,				out_channels=3      ,				down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''', '''DownEncoderBlock2D''']      ,				up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''', '''UpDecoderBlock2D''']      ,				latent_channels=4      ,				)
           return model


    @property
    def  SCREAMING_SNAKE_CASE							(    self   :			List[str])		->Dict:





           '''simple docstring'''

           torch.manual_seed(0)
           A__   				=					CLIPTextConfig(
               bos_token_id=0      ,				eos_token_id=2      ,				hidden_size=32      ,				intermediate_size=37      ,				layer_norm_eps=1e-05      ,				num_attention_heads=4      ,				num_hidden_layers=5      ,				pad_token_id=1      ,				vocab_size=1_000      ,				hidden_act='''gelu'''      ,				projection_dim=512      ,				)
           return CLIPTextModel(UpperCAmelCase__)


    def  SCREAMING_SNAKE_CASE							(    self   :			Union[str, Any])		->Tuple:





           '''simple docstring'''

           A__   				=					'''cpu'''  # ensure determinism for the device-dependent torch.Generator
           A__   				=					self.dummy_cond_unet_upscale
           A__   				=					DDPMScheduler()
           A__   				=					DDIMScheduler(prediction_type='''v_prediction''')
           A__   				=					self.dummy_vae
           A__   				=					self.dummy_text_encoder
           A__   				=					CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip''')

           A__   				=					self.dummy_image.cpu().permute(0      ,				2      ,				3      ,				1)[0]
           A__   				=					Image.fromarray(np.uinta(UpperCAmelCase__)).convert('''RGB''').resize((64, 64))

           # make sure here that pndm scheduler skips prk
           A__   				=					StableDiffusionUpscalePipeline(
               unet=UpperCAmelCase__      ,				low_res_scheduler=UpperCAmelCase__      ,				scheduler=UpperCAmelCase__      ,				vae=UpperCAmelCase__      ,				text_encoder=UpperCAmelCase__      ,				tokenizer=UpperCAmelCase__      ,				max_noise_level=350      ,				)
           A__   				=					sd_pipe.to(UpperCAmelCase__)
           sd_pipe.set_progress_bar_config(disable=UpperCAmelCase__)

           A__   				=					'''A painting of a squirrel eating a burger'''
           A__   				=					torch.Generator(device=UpperCAmelCase__).manual_seed(0)
           A__   				=					sd_pipe(
               [prompt]      ,				image=UpperCAmelCase__      ,				generator=UpperCAmelCase__      ,				guidance_scale=6.0      ,				noise_level=20      ,				num_inference_steps=2      ,				output_type='''np'''      ,				)

           A__   				=					output.images

           A__   				=					torch.Generator(device=UpperCAmelCase__).manual_seed(0)
           A__   				=					sd_pipe(
               [prompt]      ,				image=UpperCAmelCase__      ,				generator=UpperCAmelCase__      ,				guidance_scale=6.0      ,				noise_level=20      ,				num_inference_steps=2      ,				output_type='''np'''      ,				return_dict=UpperCAmelCase__      ,				)[0]

           A__   				=					image[0, -3:, -3:, -1]
           A__   				=					image_from_tuple[0, -3:, -3:, -1]

           A__   				=					low_res_image.size[0] * 4
           assert image.shape == (1, expected_height_width, expected_height_width, 3)
           A__   				=					np.array([0.3113, 0.3910, 0.4272, 0.4859, 0.5061, 0.4652, 0.5362, 0.5715, 0.5661])

           assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
           assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2


    def  SCREAMING_SNAKE_CASE							(    self   :			Union[str, Any])		->int:





           '''simple docstring'''

           A__   				=					'''cpu'''  # ensure determinism for the device-dependent torch.Generator
           A__   				=					self.dummy_cond_unet_upscale
           A__   				=					DDPMScheduler()
           A__   				=					DDIMScheduler(prediction_type='''v_prediction''')
           A__   				=					self.dummy_vae
           A__   				=					self.dummy_text_encoder
           A__   				=					CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip''')

           A__   				=					self.dummy_image.cpu().permute(0      ,				2      ,				3      ,				1)[0]
           A__   				=					Image.fromarray(np.uinta(UpperCAmelCase__)).convert('''RGB''').resize((64, 64))

           # make sure here that pndm scheduler skips prk
           A__   				=					StableDiffusionUpscalePipeline(
               unet=UpperCAmelCase__      ,				low_res_scheduler=UpperCAmelCase__      ,				scheduler=UpperCAmelCase__      ,				vae=UpperCAmelCase__      ,				text_encoder=UpperCAmelCase__      ,				tokenizer=UpperCAmelCase__      ,				max_noise_level=350      ,				)
           A__   				=					sd_pipe.to(UpperCAmelCase__)
           sd_pipe.set_progress_bar_config(disable=UpperCAmelCase__)

           A__   				=					'''A painting of a squirrel eating a burger'''
           A__   				=					sd_pipe(
               2 * [prompt]      ,				image=2 * [low_res_image]      ,				guidance_scale=6.0      ,				noise_level=20      ,				num_inference_steps=2      ,				output_type='''np'''      ,				)
           A__   				=					output.images
           assert image.shape[0] == 2

           A__   				=					torch.Generator(device=UpperCAmelCase__).manual_seed(0)
           A__   				=					sd_pipe(
               [prompt]      ,				image=UpperCAmelCase__      ,				generator=UpperCAmelCase__      ,				num_images_per_prompt=2      ,				guidance_scale=6.0      ,				noise_level=20      ,				num_inference_steps=2      ,				output_type='''np'''      ,				)
           A__   				=					output.images
           assert image.shape[0] == 2





    @unittest.skipIf(torch_device != '''cuda'''      ,				'''This test requires a GPU''')
    def  SCREAMING_SNAKE_CASE							(    self   :			Any)		->str:





           '''simple docstring'''

           A__   				=					self.dummy_cond_unet_upscale
           A__   				=					DDPMScheduler()
           A__   				=					DDIMScheduler(prediction_type='''v_prediction''')
           A__   				=					self.dummy_vae
           A__   				=					self.dummy_text_encoder
           A__   				=					CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip''')

           A__   				=					self.dummy_image.cpu().permute(0      ,				2      ,				3      ,				1)[0]
           A__   				=					Image.fromarray(np.uinta(UpperCAmelCase__)).convert('''RGB''').resize((64, 64))

           # put models in fp16, except vae as it overflows in fp16
           A__   				=					unet.half()
           A__   				=					text_encoder.half()

           # make sure here that pndm scheduler skips prk
           A__   				=					StableDiffusionUpscalePipeline(
               unet=UpperCAmelCase__      ,				low_res_scheduler=UpperCAmelCase__      ,				scheduler=UpperCAmelCase__      ,				vae=UpperCAmelCase__      ,				text_encoder=UpperCAmelCase__      ,				tokenizer=UpperCAmelCase__      ,				max_noise_level=350      ,				)
           A__   				=					sd_pipe.to(UpperCAmelCase__)
           sd_pipe.set_progress_bar_config(disable=UpperCAmelCase__)

           A__   				=					'''A painting of a squirrel eating a burger'''
           A__   				=					torch.manual_seed(0)
           A__   				=					sd_pipe(
               [prompt]      ,				image=UpperCAmelCase__      ,				generator=UpperCAmelCase__      ,				num_inference_steps=2      ,				output_type='''np'''      ,				).images

           A__   				=					low_res_image.size[0] * 4
           assert image.shape == (1, expected_height_width, expected_height_width, 3)


@slow
@require_torch_gpu
class        UpperCamelCase_					(				unittest.TestCase  ):

    '''simple docstring'''






    def  SCREAMING_SNAKE_CASE							(    self   :			Dict)		->str:





           '''simple docstring'''

           super().tearDown()
           gc.collect()
           torch.cuda.empty_cache()


    def  SCREAMING_SNAKE_CASE							(    self   :			str)		->Any:





           '''simple docstring'''

           A__   				=					load_image(
               '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
               '''/sd2-upscale/low_res_cat.png''')
           A__   				=					load_numpy(
               '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale'''
               '''/upsampled_cat.npy''')

           A__   				=					'''stabilityai/stable-diffusion-x4-upscaler'''
           A__   				=					StableDiffusionUpscalePipeline.from_pretrained(UpperCAmelCase__)
           pipe.to(UpperCAmelCase__)
           pipe.set_progress_bar_config(disable=UpperCAmelCase__)
           pipe.enable_attention_slicing()

           A__   				=					'''a cat sitting on a park bench'''

           A__   				=					torch.manual_seed(0)
           A__   				=					pipe(
               prompt=UpperCAmelCase__      ,				image=UpperCAmelCase__      ,				generator=UpperCAmelCase__      ,				output_type='''np'''      ,				)
           A__   				=					output.images[0]

           assert image.shape == (512, 512, 3)
           assert np.abs(expected_image - image).max() < 1e-3


    def  SCREAMING_SNAKE_CASE							(    self   :			int)		->str:





           '''simple docstring'''

           A__   				=					load_image(
               '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
               '''/sd2-upscale/low_res_cat.png''')
           A__   				=					load_numpy(
               '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale'''
               '''/upsampled_cat_fp16.npy''')

           A__   				=					'''stabilityai/stable-diffusion-x4-upscaler'''
           A__   				=					StableDiffusionUpscalePipeline.from_pretrained(
               UpperCAmelCase__      ,				torch_dtype=torch.floataa      ,				)
           pipe.to(UpperCAmelCase__)
           pipe.set_progress_bar_config(disable=UpperCAmelCase__)
           pipe.enable_attention_slicing()

           A__   				=					'''a cat sitting on a park bench'''

           A__   				=					torch.manual_seed(0)
           A__   				=					pipe(
               prompt=UpperCAmelCase__      ,				image=UpperCAmelCase__      ,				generator=UpperCAmelCase__      ,				output_type='''np'''      ,				)
           A__   				=					output.images[0]

           assert image.shape == (512, 512, 3)
           assert np.abs(expected_image - image).max() < 5e-1





    def  SCREAMING_SNAKE_CASE							(    self   :			Union[str, Any])		->List[Any]:





           '''simple docstring'''

           torch.cuda.empty_cache()
           torch.cuda.reset_max_memory_allocated()
           torch.cuda.reset_peak_memory_stats()

           A__   				=					load_image(
               '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
               '''/sd2-upscale/low_res_cat.png''')

           A__   				=					'''stabilityai/stable-diffusion-x4-upscaler'''
           A__   				=					StableDiffusionUpscalePipeline.from_pretrained(
               UpperCAmelCase__      ,				torch_dtype=torch.floataa      ,				)
           pipe.to(UpperCAmelCase__)
           pipe.set_progress_bar_config(disable=UpperCAmelCase__)
           pipe.enable_attention_slicing(1)
           pipe.enable_sequential_cpu_offload()

           A__   				=					'''a cat sitting on a park bench'''

           A__   				=					torch.manual_seed(0)
           A__   				=					pipe(
               prompt=UpperCAmelCase__      ,				image=UpperCAmelCase__      ,				generator=UpperCAmelCase__      ,				num_inference_steps=5      ,				output_type='''np'''      ,				)

           A__   				=					torch.cuda.max_memory_allocated()
           # make sure that less than 2.9 GB is allocated
           assert mem_bytes < 2.9 * 10**9


