
"""simple docstring"""

import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config

@pytest.fixture(scope='session' )
def     lowercase__						(					) ->      List[Any]:






    """simple docstring"""




    UpperCAmelCase									=					10
    UpperCAmelCase									=					datasets.Features(
        {
            'tokens': datasets.Sequence(datasets.Value('string' ) ),
            'labels': datasets.Sequence(datasets.ClassLabel(names=['negative', 'positive'] ) ),
            'answers': datasets.Sequence(
                {
                    'text': datasets.Value('string' ),
                    'answer_start': datasets.Value('int32' ),
                } ),
            'id': datasets.Value('int64' ),
        } )
    UpperCAmelCase									=					datasets.Dataset.from_dict(
        {
            'tokens': [['foo'] * 5] * n,
            'labels': [[1] * 5] * n,
            'answers': [{'answer_start': [97], 'text': ['1976']}] * 10,
            'id': list(range(lowerCamelCase__ ) ),
        } ,				features=lowerCamelCase__ ,				)
    return dataset

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Optional[Any] ,				lowerCAmelCase				:					Union[str, Any] ) ->      Dict:






    """simple docstring"""




    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'file.arrow' )
    dataset.map(cache_file_name=lowerCamelCase__ )
    return filename


# FILE_CONTENT + files


SCREAMING_SNAKE_CASE_    					=						'''\
    Text data.
    Second line of data.'''

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Optional[int] ) ->      Union[str, Any]:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "file.txt"
    UpperCAmelCase									=					FILE_CONTENT
    with open(lowerCamelCase__ ,				'w' ) as f:
        f.write(lowerCamelCase__ )
    return filename

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Union[str, Any] ) ->      Dict:






    """simple docstring"""




    import bza

    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "file.txt.bz2"
    UpperCAmelCase									=					bytes(lowerCamelCase__ ,				'utf-8' )
    with bza.open(lowerCamelCase__ ,				'wb' ) as f:
        f.write(lowerCamelCase__ )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Any ) ->      Tuple:






    """simple docstring"""




    import gzip

    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'file.txt.gz' )
    UpperCAmelCase									=					bytes(lowerCamelCase__ ,				'utf-8' )
    with gzip.open(lowerCamelCase__ ,				'wb' ) as f:
        f.write(lowerCamelCase__ )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					str ) ->      Optional[int]:






    """simple docstring"""




    if datasets.config.LZ4_AVAILABLE:
        import lza.frame

        UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "file.txt.lz4"
        UpperCAmelCase									=					bytes(lowerCamelCase__ ,				'utf-8' )
        with lza.frame.open(lowerCamelCase__ ,				'wb' ) as f:
            f.write(lowerCamelCase__ )
        return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Dict ,				lowerCAmelCase				:					Optional[int] ) ->      Optional[int]:






    """simple docstring"""




    if datasets.config.PY7ZR_AVAILABLE:
        import pyazr

        UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "file.txt.7z"
        with pyazr.SevenZipFile(lowerCamelCase__ ,				'w' ) as archive:
            archive.write(lowerCamelCase__ ,				arcname=os.path.basename(lowerCamelCase__ ) )
        return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					int ,				lowerCAmelCase				:					Optional[Any] ) ->      List[Any]:






    """simple docstring"""




    import tarfile

    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "file.txt.tar"
    with tarfile.TarFile(lowerCamelCase__ ,				'w' ) as f:
        f.add(lowerCamelCase__ ,				arcname=os.path.basename(lowerCamelCase__ ) )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Optional[Any] ) ->      List[str]:






    """simple docstring"""




    import lzma

    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "file.txt.xz"
    UpperCAmelCase									=					bytes(lowerCamelCase__ ,				'utf-8' )
    with lzma.open(lowerCamelCase__ ,				'wb' ) as f:
        f.write(lowerCamelCase__ )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					str ,				lowerCAmelCase				:					int ) ->      Dict:






    """simple docstring"""




    import zipfile

    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "file.txt.zip"
    with zipfile.ZipFile(lowerCamelCase__ ,				'w' ) as f:
        f.write(lowerCamelCase__ ,				arcname=os.path.basename(lowerCamelCase__ ) )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Optional[int] ) ->      List[Any]:






    """simple docstring"""




    if datasets.config.ZSTANDARD_AVAILABLE:
        import zstandard as zstd

        UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "file.txt.zst"
        UpperCAmelCase									=					bytes(lowerCamelCase__ ,				'utf-8' )
        with zstd.open(lowerCamelCase__ ,				'wb' ) as f:
            f.write(lowerCamelCase__ )
        return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					List[str] ) ->      Dict:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "file.xml"
    UpperCAmelCase									=					textwrap.dedent(
        '\\n    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n    <tmx version=\"1.4\">\n      <header segtype=\"sentence\" srclang=\"ca\" />\n      <body>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>' )
    with open(lowerCamelCase__ ,				'w' ) as f:
        f.write(lowerCamelCase__ )
    return filename


SCREAMING_SNAKE_CASE_    					=						[
    {'''col_1''': '''0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''3''', '''col_2''': 3, '''col_3''': 3.0},
]
SCREAMING_SNAKE_CASE_    					=						[
    {'''col_1''': '''4''', '''col_2''': 4, '''col_3''': 4.0},
    {'''col_1''': '''5''', '''col_2''': 5, '''col_3''': 5.0},
]
SCREAMING_SNAKE_CASE_    					=						{
    '''col_1''': ['''0''', '''1''', '''2''', '''3'''],
    '''col_2''': [0, 1, 2, 3],
    '''col_3''': [0.0, 1.0, 2.0, 3.0],
}

SCREAMING_SNAKE_CASE_    					=						[
    {'''col_3''': 0.0, '''col_1''': '''0''', '''col_2''': 0},
    {'''col_3''': 1.0, '''col_1''': '''1''', '''col_2''': 1},
]

SCREAMING_SNAKE_CASE_    					=						[
    {'''col_1''': '''s0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''s1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''s2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''s3''', '''col_2''': 3, '''col_3''': 3.0},
]

@pytest.fixture(scope='session' )
def     lowercase__						(					) ->      Union[str, Any]:






    """simple docstring"""




    return DATA_DICT_OF_LISTS

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Optional[Any] ) ->      List[str]:






    """simple docstring"""




    UpperCAmelCase									=					datasets.Dataset.from_dict(lowerCamelCase__ )
    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset.arrow' )
    dataset.map(cache_file_name=lowerCamelCase__ )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					List[Any] ) ->      List[str]:






    """simple docstring"""




    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset.sqlite' )
    with contextlib.closing(sqlitea.connect(lowerCamelCase__ ) ) as con:
        UpperCAmelCase									=					con.cursor()
        cur.execute('CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)' )
        for item in DATA:
            cur.execute('INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)' ,				tuple(item.values() ) )
        con.commit()
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Union[str, Any] ) ->      Dict:






    """simple docstring"""




    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset.csv' )
    with open(lowerCamelCase__ ,				'w' ,				newline='' ) as f:
        UpperCAmelCase									=					csv.DictWriter(lowerCamelCase__ ,				fieldnames=['col_1', 'col_2', 'col_3'] )
        writer.writeheader()
        for item in DATA:
            writer.writerow(lowerCamelCase__ )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					str ) ->      Dict:






    """simple docstring"""




    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset2.csv' )
    with open(lowerCamelCase__ ,				'w' ,				newline='' ) as f:
        UpperCAmelCase									=					csv.DictWriter(lowerCamelCase__ ,				fieldnames=['col_1', 'col_2', 'col_3'] )
        writer.writeheader()
        for item in DATA:
            writer.writerow(lowerCamelCase__ )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Optional[Any] ,				lowerCAmelCase				:					List[Any] ) ->      str:






    """simple docstring"""




    import bza

    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset.csv.bz2"
    with open(lowerCamelCase__ ,				'rb' ) as f:
        UpperCAmelCase									=					f.read()
    # data = bytes(FILE_CONTENT, "utf-8")
    with bza.open(lowerCamelCase__ ,				'wb' ) as f:
        f.write(lowerCamelCase__ )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Tuple ,				lowerCAmelCase				:					List[str] ,				lowerCAmelCase				:					Optional[Any] ) ->      Tuple:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset.csv.zip"
    with zipfile.ZipFile(lowerCamelCase__ ,				'w' ) as f:
        f.write(lowerCamelCase__ ,				arcname=os.path.basename(lowerCamelCase__ ) )
        f.write(lowerCamelCase__ ,				arcname=os.path.basename(lowerCamelCase__ ) )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					List[Any] ,				lowerCAmelCase				:					List[str] ,				lowerCAmelCase				:					Optional[int] ) ->      Any:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset.csv.zip"
    with zipfile.ZipFile(lowerCamelCase__ ,				'w' ) as f:
        f.write(lowerCamelCase__ ,				arcname=os.path.basename(csv_path.replace('.csv' ,				'.CSV' ) ) )
        f.write(lowerCamelCase__ ,				arcname=os.path.basename(csva_path.replace('.csv' ,				'.CSV' ) ) )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					str ,				lowerCAmelCase				:					List[Any] ,				lowerCAmelCase				:					int ) ->      Union[str, Any]:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset_with_dir.csv.zip"
    with zipfile.ZipFile(lowerCamelCase__ ,				'w' ) as f:
        f.write(lowerCamelCase__ ,				arcname=os.path.join('main_dir' ,				os.path.basename(lowerCamelCase__ ) ) )
        f.write(lowerCamelCase__ ,				arcname=os.path.join('main_dir' ,				os.path.basename(lowerCamelCase__ ) ) )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Tuple ) ->      str:






    """simple docstring"""




    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset.parquet' )
    UpperCAmelCase									=					pa.schema(
        {
            'col_1': pa.string(),
            'col_2': pa.intaa(),
            'col_3': pa.floataa(),
        } )
    with open(lowerCamelCase__ ,				'wb' ) as f:
        UpperCAmelCase									=					pq.ParquetWriter(lowerCamelCase__ ,				schema=lowerCamelCase__ )
        UpperCAmelCase									=					pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(lowerCamelCase__ ) )] for k in DATA[0]} ,				schema=lowerCamelCase__ )
        writer.write_table(lowerCamelCase__ )
        writer.close()
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					int ) ->      Optional[int]:






    """simple docstring"""




    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset.json' )
    UpperCAmelCase									=					{"data": DATA}
    with open(lowerCamelCase__ ,				'w' ) as f:
        json.dump(lowerCamelCase__ ,				lowerCamelCase__ )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					List[Any] ) ->      Dict:






    """simple docstring"""




    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset.json' )
    UpperCAmelCase									=					{"data": DATA_DICT_OF_LISTS}
    with open(lowerCamelCase__ ,				'w' ) as f:
        json.dump(lowerCamelCase__ ,				lowerCamelCase__ )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Tuple ) ->      List[str]:






    """simple docstring"""




    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset.jsonl' )
    with open(lowerCamelCase__ ,				'w' ) as f:
        for item in DATA:
            f.write(json.dumps(lowerCamelCase__ ) + '\n' )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Optional[Any] ) ->      int:






    """simple docstring"""




    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset2.jsonl' )
    with open(lowerCamelCase__ ,				'w' ) as f:
        for item in DATA:
            f.write(json.dumps(lowerCamelCase__ ) + '\n' )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					List[str] ) ->      int:






    """simple docstring"""




    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset_312.jsonl' )
    with open(lowerCamelCase__ ,				'w' ) as f:
        for item in DATA_312:
            f.write(json.dumps(lowerCamelCase__ ) + '\n' )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					int ) ->      Union[str, Any]:






    """simple docstring"""




    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset-str.jsonl' )
    with open(lowerCamelCase__ ,				'w' ) as f:
        for item in DATA_STR:
            f.write(json.dumps(lowerCamelCase__ ) + '\n' )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Dict ,				lowerCAmelCase				:					str ) ->      Dict:






    """simple docstring"""




    import gzip

    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset.txt.gz' )
    with open(lowerCamelCase__ ,				'rb' ) as orig_file:
        with gzip.open(lowerCamelCase__ ,				'wb' ) as zipped_file:
            zipped_file.writelines(lowerCamelCase__ )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Tuple ,				lowerCAmelCase				:					Tuple ) ->      List[Any]:






    """simple docstring"""




    import gzip

    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset.jsonl.gz' )
    with open(lowerCamelCase__ ,				'rb' ) as orig_file:
        with gzip.open(lowerCamelCase__ ,				'wb' ) as zipped_file:
            zipped_file.writelines(lowerCamelCase__ )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Dict ,				lowerCAmelCase				:					int ,				lowerCAmelCase				:					List[str] ) ->      Optional[Any]:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset.jsonl.zip"
    with zipfile.ZipFile(lowerCamelCase__ ,				'w' ) as f:
        f.write(lowerCamelCase__ ,				arcname=os.path.basename(lowerCamelCase__ ) )
        f.write(lowerCamelCase__ ,				arcname=os.path.basename(lowerCamelCase__ ) )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					int ,				lowerCAmelCase				:					Dict ,				lowerCAmelCase				:					Tuple ,				lowerCAmelCase				:					Optional[int] ) ->      Union[str, Any]:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset_nested.jsonl.zip"
    with zipfile.ZipFile(lowerCamelCase__ ,				'w' ) as f:
        f.write(lowerCamelCase__ ,				arcname=os.path.join('nested' ,				os.path.basename(lowerCamelCase__ ) ) )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Union[str, Any] ,				lowerCAmelCase				:					Tuple ,				lowerCAmelCase				:					str ) ->      Optional[int]:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset_with_dir.jsonl.zip"
    with zipfile.ZipFile(lowerCamelCase__ ,				'w' ) as f:
        f.write(lowerCamelCase__ ,				arcname=os.path.join('main_dir' ,				os.path.basename(lowerCamelCase__ ) ) )
        f.write(lowerCamelCase__ ,				arcname=os.path.join('main_dir' ,				os.path.basename(lowerCamelCase__ ) ) )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Dict ,				lowerCAmelCase				:					Dict ,				lowerCAmelCase				:					int ) ->      Dict:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset.jsonl.tar"
    with tarfile.TarFile(lowerCamelCase__ ,				'w' ) as f:
        f.add(lowerCamelCase__ ,				arcname=os.path.basename(lowerCamelCase__ ) )
        f.add(lowerCamelCase__ ,				arcname=os.path.basename(lowerCamelCase__ ) )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Dict ,				lowerCAmelCase				:					str ,				lowerCAmelCase				:					Any ,				lowerCAmelCase				:					List[str] ) ->      Tuple:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset_nested.jsonl.tar"
    with tarfile.TarFile(lowerCamelCase__ ,				'w' ) as f:
        f.add(lowerCamelCase__ ,				arcname=os.path.join('nested' ,				os.path.basename(lowerCamelCase__ ) ) )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					int ) ->      Tuple:






    """simple docstring"""




    UpperCAmelCase									=					["0", "1", "2", "3"]
    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset.txt' )
    with open(lowerCamelCase__ ,				'w' ) as f:
        for item in data:
            f.write(item + '\n' )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					str ) ->      Optional[Any]:






    """simple docstring"""




    UpperCAmelCase									=					["0", "1", "2", "3"]
    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset2.txt' )
    with open(lowerCamelCase__ ,				'w' ) as f:
        for item in data:
            f.write(item + '\n' )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					str ) ->      Union[str, Any]:






    """simple docstring"""




    UpperCAmelCase									=					["0", "1", "2", "3"]
    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset.abc"
    with open(lowerCamelCase__ ,				'w' ) as f:
        for item in data:
            f.write(item + '\n' )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Any ,				lowerCAmelCase				:					List[str] ,				lowerCAmelCase				:					Any ) ->      Dict:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset.text.zip"
    with zipfile.ZipFile(lowerCamelCase__ ,				'w' ) as f:
        f.write(lowerCamelCase__ ,				arcname=os.path.basename(lowerCamelCase__ ) )
        f.write(lowerCamelCase__ ,				arcname=os.path.basename(lowerCamelCase__ ) )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					int ,				lowerCAmelCase				:					int ,				lowerCAmelCase				:					List[Any] ) ->      Dict:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset_with_dir.text.zip"
    with zipfile.ZipFile(lowerCamelCase__ ,				'w' ) as f:
        f.write(lowerCamelCase__ ,				arcname=os.path.join('main_dir' ,				os.path.basename(lowerCamelCase__ ) ) )
        f.write(lowerCamelCase__ ,				arcname=os.path.join('main_dir' ,				os.path.basename(lowerCamelCase__ ) ) )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Optional[Any] ,				lowerCAmelCase				:					Any ,				lowerCAmelCase				:					Union[str, Any] ) ->      Any:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset.ext.zip"
    with zipfile.ZipFile(lowerCamelCase__ ,				'w' ) as f:
        f.write(lowerCamelCase__ ,				arcname=os.path.basename('unsupported.ext' ) )
        f.write(lowerCamelCase__ ,				arcname=os.path.basename('unsupported_2.ext' ) )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					Dict ) ->      Any:






    """simple docstring"""




    UpperCAmelCase									=					"\n".join(['First', 'Second\u2029with Unicode new line', 'Third'] )
    UpperCAmelCase									=					str(tmp_path_factory.mktemp('data' ) / 'dataset_with_unicode_new_lines.txt' )
    with open(lowerCamelCase__ ,				'w' ,				encoding='utf-8' ) as f:
        f.write(lowerCamelCase__ )
    return path

@pytest.fixture(scope='session' )
def     lowercase__						(					) ->      Any:






    """simple docstring"""




    return os.path.join('tests' ,				'features' ,				'data' ,				'test_image_rgb.jpg' )

@pytest.fixture(scope='session' )
def     lowercase__						(					) ->      List[str]:






    """simple docstring"""




    return os.path.join('tests' ,				'features' ,				'data' ,				'test_audio_44100.wav' )

@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					List[Any] ,				lowerCAmelCase				:					List[Any] ) ->      Dict:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data' ) / "dataset.img.zip"
    with zipfile.ZipFile(lowerCamelCase__ ,				'w' ) as f:
        f.write(lowerCamelCase__ ,				arcname=os.path.basename(lowerCamelCase__ ) )
        f.write(lowerCamelCase__ ,				arcname=os.path.basename(lowerCamelCase__ ).replace('.jpg' ,				'2.jpg' ) )
    return path







@pytest.fixture(scope='session' )
def     lowercase__						(					lowerCAmelCase				:					str ) ->      Optional[int]:






    """simple docstring"""




    UpperCAmelCase									=					tmp_path_factory.mktemp('data_dir' )

    (data_dir / "subdir").mkdir()
    with open(data_dir / 'subdir' / 'train.txt' ,				'w' ) as f:
        f.write('foo\n' * 10 )
    with open(data_dir / 'subdir' / 'test.txt' ,				'w' ) as f:
        f.write('bar\n' * 10 )
    # hidden file
    with open(data_dir / 'subdir' / '.test.txt' ,				'w' ) as f:
        f.write('bar\n' * 10 )

    # hidden directory
    (data_dir / ".subdir").mkdir()
    with open(data_dir / '.subdir' / 'train.txt' ,				'w' ) as f:
        f.write('foo\n' * 10 )
    with open(data_dir / '.subdir' / 'test.txt' ,				'w' ) as f:
        f.write('bar\n' * 10 )

    return data_dir





"""simple docstring"""

import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_    					=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_    					=						{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

SCREAMING_SNAKE_CASE_    					=						{
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

SCREAMING_SNAKE_CASE_    					=						{
    '''ctrl''': 256,
}

SCREAMING_SNAKE_CASE_    					=						{
    '''Pregnancy''': 16_8629,
    '''Christianity''': 7675,
    '''Explain''': 10_6423,
    '''Fitness''': 6_3440,
    '''Saving''': 6_3163,
    '''Ask''': 2_7171,
    '''Ass''': 9_5985,
    '''Joke''': 16_3509,
    '''Questions''': 4_5622,
    '''Thoughts''': 4_9605,
    '''Retail''': 5_2342,
    '''Feminism''': 16_4338,
    '''Writing''': 1_1992,
    '''Atheism''': 19_2263,
    '''Netflix''': 4_8616,
    '''Computing''': 3_9639,
    '''Opinion''': 4_3213,
    '''Alone''': 4_4967,
    '''Funny''': 5_8917,
    '''Gaming''': 4_0358,
    '''Human''': 4088,
    '''India''': 1331,
    '''Joker''': 7_7138,
    '''Diet''': 3_6206,
    '''Legal''': 1_1859,
    '''Norman''': 4939,
    '''Tip''': 7_2689,
    '''Weight''': 5_2343,
    '''Movies''': 4_6273,
    '''Running''': 2_3425,
    '''Science''': 2090,
    '''Horror''': 3_7793,
    '''Confession''': 6_0572,
    '''Finance''': 1_2250,
    '''Politics''': 1_6360,
    '''Scary''': 19_1985,
    '''Support''': 1_2654,
    '''Technologies''': 3_2516,
    '''Teenage''': 6_6160,
    '''Event''': 3_2769,
    '''Learned''': 6_7460,
    '''Notion''': 18_2770,
    '''Wikipedia''': 3_7583,
    '''Books''': 6665,
    '''Extract''': 7_6050,
    '''Confessions''': 10_2701,
    '''Conspiracy''': 7_5932,
    '''Links''': 6_3674,
    '''Narcissus''': 15_0425,
    '''Relationship''': 5_4766,
    '''Relationships''': 13_4796,
    '''Reviews''': 4_1671,
    '''News''': 4256,
    '''Translation''': 2_6820,
    '''multilingual''': 12_8406,
}




def     lowercase__						(					lowerCAmelCase				:					int ) ->      int:






    """simple docstring"""




    UpperCAmelCase									=					set()
    UpperCAmelCase									=					word[0]
    for char in word[1:]:
        pairs.add((prev_char, char) )
        UpperCAmelCase									=					char

    UpperCAmelCase									=					set(lowerCAmelCase )
    return pairs







class      _UpperCAmelCase							(		SCREAMING_SNAKE_CASE_							):
      __SCREAMING_SNAKE_CASE   :							Optional[Any]							      =	VOCAB_FILES_NAMES
      __SCREAMING_SNAKE_CASE   :							List[str]							      =	PRETRAINED_VOCAB_FILES_MAP
      __SCREAMING_SNAKE_CASE   :							List[Any]							      =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __SCREAMING_SNAKE_CASE   :							Union[str, Any]							      =	CONTROL_CODES




      def __init__(							self   ,       lowercase_   ,       lowercase_   ,       lowercase_="<unk>"   ,       **lowercase_							)						->					Optional[int]:
          super().__init__(unk_token=lowercase_   ,       **lowercase_							)

          with open(lowercase_   ,       encoding='utf-8'							) as vocab_handle:
              UpperCAmelCase									=					json.load(lowercase_							)
          UpperCAmelCase									=					{v: k for k, v in self.encoder.items()}
          with open(lowercase_   ,       encoding='utf-8'							) as merges_handle:
              UpperCAmelCase									=					merges_handle.read().split('\n'							)[1:-1]
          UpperCAmelCase									=					[tuple(merge.split()							) for merge in merges]
          UpperCAmelCase									=					dict(zip(lowercase_   ,       range(len(lowercase_							)							)							)							)
          UpperCAmelCase									=					{}




      @property
      def     a_      (							self							)						->					Union[str, Any]:
          return len(self.encoder							)




      def     a_      (							self							)						->					Optional[Any]:
          return dict(self.encoder   ,       **self.added_tokens_encoder							)




      def     a_      (							self   ,       lowercase_							)						->					int:
          if token in self.cache:
              return self.cache[token]
          UpperCAmelCase									=					tuple(lowercase_							)
          UpperCAmelCase									=					tuple(list(word[:-1]							) + [word[-1] + '</w>']							)
          UpperCAmelCase									=					get_pairs(lowercase_							)

          if not pairs:
              return token

          while True:
              UpperCAmelCase									=					min(lowercase_   ,       key=lambda lowercase_							: self.bpe_ranks.get(lowercase_   ,       float('inf'							)							)							)
              if bigram not in self.bpe_ranks:
                  break
              UpperCAmelCase			,							UpperCAmelCase									=					bigram
              UpperCAmelCase									=					[]
              UpperCAmelCase									=					0
              while i < len(lowercase_							):
                  try:
                      UpperCAmelCase									=					word.index(lowercase_   ,       lowercase_							)
                  except ValueError:
                      new_word.extend(word[i:]							)
                      break
                  else:
                      new_word.extend(word[i:j]							)
                      UpperCAmelCase									=					j

                  if word[i] == first and i < len(lowercase_							) - 1 and word[i + 1] == second:
                      new_word.append(first + second							)
                      i += 2
                  else:
                      new_word.append(word[i]							)
                      i += 1
              UpperCAmelCase									=					tuple(lowercase_							)
              UpperCAmelCase									=					new_word
              if len(lowercase_							) == 1:
                  break
              else:
                  UpperCAmelCase									=					get_pairs(lowercase_							)
          UpperCAmelCase									=					'@@ '.join(lowercase_							)
          UpperCAmelCase									=					word[:-4]
          UpperCAmelCase									=					word
          return word




      def     a_      (							self   ,       lowercase_							)						->					List[Any]:
          UpperCAmelCase									=					[]

          UpperCAmelCase									=					re.findall(R'\S+\n?'   ,       lowercase_							)

          for token in words:
              split_tokens.extend(list(self.bpe(lowercase_							).split(' '							)							)							)
          return split_tokens




      def     a_      (							self   ,       lowercase_							)						->					Tuple:
          return self.encoder.get(lowercase_   ,       self.encoder.get(self.unk_token							)							)




      def     a_      (							self   ,       lowercase_							)						->					Optional[Any]:
          return self.decoder.get(lowercase_   ,       self.unk_token							)




      def     a_      (							self   ,       lowercase_							)						->					str:
          UpperCAmelCase									=					' '.join(lowercase_							).replace('@@ '   ,       ''							).strip()
          return out_string




      def     a_      (							self   ,       lowercase_   ,       lowercase_ = None							)						->					Tuple[str]:
          if not os.path.isdir(lowercase_							):
              logger.error(F"Vocabulary path ({save_directory}) should be a directory"							)
              return
          UpperCAmelCase									=					os.path.join(
              lowercase_   ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)
          UpperCAmelCase									=					os.path.join(
              lowercase_   ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']							)

          with open(lowercase_   ,       'w'   ,       encoding='utf-8'							) as f:
              f.write(json.dumps(self.encoder   ,       indent=2   ,       sort_keys=lowercase_   ,       ensure_ascii=lowercase_							) + '\n'							)

          UpperCAmelCase									=					0
          with open(lowercase_   ,       'w'   ,       encoding='utf-8'							) as writer:
              writer.write('#version: 0.2\n'							)
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,       key=lambda lowercase_							: kv[1]							):
                  if index != token_index:
                      logger.warning(
                          F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                          ' Please check that the tokenizer is not corrupted!'							)
                      UpperCAmelCase									=					token_index
                  writer.write(' '.join(lowercase_							) + '\n'							)
                  index += 1

          return vocab_file, merge_file

      # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
      #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
      #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
      #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
      #     return ''.join(tokens_generated_so_far)



