



import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_lowercase							  =		"▁"

_lowercase							  =		get_tests_dir("fixtures/test_sentencepiece.model")



@require_sentencepiece
@require_tokenizers
class     _UpperCAmelCase			(       UpperCAmelCase_			,      unittest.TestCase     ):
     UpperCamelCase__		    =				BigBirdTokenizer
     UpperCamelCase__		    =				BigBirdTokenizerFast
     UpperCamelCase__		    =				True
     UpperCamelCase__		    =				True




     def 		snake_case_		(				self):
         super().setUp()

         A__		 =					self.tokenizer_class(_snake_case ,						keep_accents=_snake_case)
         tokenizer.save_pretrained(self.tmpdirname)




     def 		snake_case_		(				self):
         A__		 =					'''<s>'''
         A__		 =					1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(_snake_case) ,						_snake_case)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(_snake_case) ,						_snake_case)




     def 		snake_case_		(				self):
         A__		 =					list(self.get_tokenizer().get_vocab().keys())

         self.assertEqual(vocab_keys[0] ,						'''<unk>''')
         self.assertEqual(vocab_keys[1] ,						'''<s>''')
         self.assertEqual(vocab_keys[-1] ,						'''[MASK]''')
         self.assertEqual(len(_snake_case) ,						1_0_0_4)




     def 		snake_case_		(				self):
         self.assertEqual(self.get_tokenizer().vocab_size ,						1_0_0_0)




     def 		snake_case_		(				self):


         if not self.test_rust_tokenizer:
             return

         A__		 =					self.get_tokenizer()
         A__		 =					self.get_rust_tokenizer()

         A__		 =					'''I was born in 92000, and this is falsé.'''

         A__		 =					tokenizer.tokenize(_snake_case)
         A__		 =					rust_tokenizer.tokenize(_snake_case)
         self.assertListEqual(_snake_case ,						_snake_case)

         A__		 =					tokenizer.encode(_snake_case ,						add_special_tokens=_snake_case)
         A__		 =					rust_tokenizer.encode(_snake_case ,						add_special_tokens=_snake_case)
         self.assertListEqual(_snake_case ,						_snake_case)

         A__		 =					self.get_rust_tokenizer()
         A__		 =					tokenizer.encode(_snake_case)
         A__		 =					rust_tokenizer.encode(_snake_case)
         self.assertListEqual(_snake_case ,						_snake_case)




     def 		snake_case_		(				self):
         A__		 =					BigBirdTokenizer(_snake_case ,						keep_accents=_snake_case)

         A__		 =					tokenizer.tokenize('''This is a test''')
         self.assertListEqual(_snake_case ,						['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(_snake_case) ,						[2_8_5, 4_6, 1_0, 1_7_0, 3_8_2] ,						)

         A__		 =					tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
         self.assertListEqual(
             _snake_case ,						[
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''9''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''é''',
                 '''.''',
             ] ,						)
         A__		 =					tokenizer.convert_tokens_to_ids(_snake_case)
         self.assertListEqual(
             _snake_case ,						[8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4] ,						)

         A__		 =					tokenizer.convert_ids_to_tokens(_snake_case)
         self.assertListEqual(
             _snake_case ,						[
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''<unk>''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''<unk>''',
                 '''.''',
             ] ,						)




     @cached_property
     def 		snake_case_		(				self):
         return BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''')




     @slow
     def 		snake_case_		(				self):
         A__		 =					'''Hello World!'''
         A__		 =					[6_5, 1_8_5_3_6, 2_2_6_0, 1_0_1, 6_6]

         self.assertListEqual(_snake_case ,						self.big_tokenizer.encode(_snake_case))




     @slow
     def 		snake_case_		(				self):
         A__		 =					(
             '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
             ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
         )
         # fmt: off
         A__		 =					[6_5, 8_7_1, 4_1_9, 3_5_8, 9_4_6, 9_9_1, 2_5_2_1, 4_5_2, 3_5_8, 1_3_5_7, 3_8_7, 7_7_5_1, 3_5_3_6, 1_1_2, 9_8_5, 4_5_6, 1_2_6, 8_6_5, 9_3_8, 5_4_0_0, 5_7_3_4, 4_5_8, 1_3_6_8, 4_6_7, 7_8_6, 2_4_6_2, 5_2_4_6, 1_1_5_9, 6_3_3, 8_6_5, 4_5_1_9, 4_5_7, 5_8_2, 8_5_2, 2_5_5_7, 4_2_7, 9_1_6, 5_0_8, 4_0_5, 3_4_3_2_4, 4_9_7, 3_9_1, 4_0_8, 1_1_3_4_2, 1_2_4_4, 3_8_5, 1_0_0, 9_3_8, 9_8_5, 4_5_6, 5_7_4, 3_6_2, 1_2_5_9_7, 3_2_0_0, 3_1_2_9, 1_1_7_2, 6_6]  # noqa: E231
         # fmt: on
         self.assertListEqual(_snake_case ,						self.big_tokenizer.encode(_snake_case))




     @require_torch
     @slow
     def 		snake_case_		(				self):
         import torch

         from transformers import BigBirdConfig, BigBirdModel

         # Build sequence
         A__		 =					list(self.big_tokenizer.get_vocab().keys())[:1_0]
         A__		 =					''' '''.join(_snake_case)
         A__		 =					self.big_tokenizer.encode_plus(_snake_case ,						return_tensors='''pt''' ,						return_token_type_ids=_snake_case)
         A__		 =					self.big_tokenizer.batch_encode_plus(
             [sequence + ''' ''' + sequence] ,						return_tensors='''pt''' ,						return_token_type_ids=_snake_case)

         A__		 =					BigBirdConfig(attention_type='''original_full''')
         A__		 =					BigBirdModel(_snake_case)

         assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

         with torch.no_grad():
             model(**_snake_case)
             model(**_snake_case)




     @slow
     def 		snake_case_		(				self):
         A__		 =					BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''')
         A__		 =					tokenizer.decode(tokenizer('''Paris is the [MASK].''').input_ids)

         self.assertTrue(decoded_text == '''[CLS] Paris is the[MASK].[SEP]''')




     @slow
     def 		snake_case_		(				self):
         A__		 =					{'''input_ids''': [[6_5, 3_9_2_8_6, 4_5_8, 3_6_3_3_5, 2_0_0_1, 4_5_6, 1_3_0_7_3, 1_3_2_6_6, 4_5_5, 1_1_3, 7_7_4_6, 1_7_4_1, 1_1_1_5_7, 3_9_1, 1_3_0_7_3, 1_3_2_6_6, 4_5_5, 1_1_3, 3_9_6_7, 3_5_4_1_2, 1_1_3, 4_9_3_6, 1_0_9, 3_8_7_0, 2_3_7_7, 1_1_3, 3_0_0_8_4, 4_5_7_2_0, 4_5_8, 1_3_4, 1_7_4_9_6, 1_1_2, 5_0_3, 1_1_6_7_2, 1_1_3, 1_1_8, 1_1_2, 5_6_6_5, 1_3_3_4_7, 3_8_6_8_7, 1_1_2, 1_4_9_6, 3_1_3_8_9, 1_1_2, 3_2_6_8, 4_7_2_6_4, 1_3_4, 9_6_2, 1_1_2, 1_6_3_7_7, 8_0_3_5, 2_3_1_3_0, 4_3_0, 1_2_1_6_9, 1_5_5_1_8, 2_8_5_9_2, 4_5_8, 1_4_6, 4_1_6_9_7, 1_0_9, 3_9_1, 1_2_1_6_9, 1_5_5_1_8, 1_6_6_8_9, 4_5_8, 1_4_6, 4_1_3_5_8, 1_0_9, 4_5_2, 7_2_6, 4_0_3_4, 1_1_1, 7_6_3, 3_5_4_1_2, 5_0_8_2, 3_8_8, 1_9_0_3, 1_1_1, 9_0_5_1, 3_9_1, 2_8_7_0, 4_8_9_1_8, 1_9_0_0, 1_1_2_3, 5_5_0, 9_9_8, 1_1_2, 9_5_8_6, 1_5_9_8_5, 4_5_5, 3_9_1, 4_1_0, 2_2_9_5_5, 3_7_6_3_6, 1_1_4, 6_6], [6_5, 4_4_8, 1_7_4_9_6, 4_1_9, 3_6_6_3, 3_8_5, 7_6_3, 1_1_3, 2_7_5_3_3, 2_8_7_0, 3_2_8_3, 1_3_0_4_3, 1_6_3_9, 2_4_7_1_3, 5_2_3, 6_5_6, 2_4_0_1_3, 1_8_5_5_0, 2_5_2_1, 5_1_7, 2_7_0_1_4, 2_1_2_4_4, 4_2_0, 1_2_1_2, 1_4_6_5, 3_9_1, 9_2_7, 4_8_3_3, 3_8_8, 5_7_8, 1_1_7_8_6, 1_1_4, 6_6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [6_5, 4_8_4, 2_1_6_9, 7_6_8_7, 2_1_9_3_2, 1_8_1_4_6, 7_2_6, 3_6_3, 1_7_0_3_2, 3_3_9_1, 1_1_4, 6_6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=_snake_case ,						model_name='''google/bigbird-roberta-base''' ,						revision='''215c99f1600e06f83acce68422f2035b2b5c3510''' ,						)





import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  str:
    A__		 =					1_0
    A__		 =					datasets.Features(
        {
            '''tokens''': datasets.Sequence(datasets.Value('''string'''	)	),
            '''labels''': datasets.Sequence(datasets.ClassLabel(names=['''negative''', '''positive''']	)	),
            '''answers''': datasets.Sequence(
                {
                    '''text''': datasets.Value('''string'''	),
                    '''answer_start''': datasets.Value('''int32'''	),
                }	),
            '''id''': datasets.Value('''int64'''	),
        }	)
    A__		 =					datasets.Dataset.from_dict(
        {
            '''tokens''': [['''foo'''] * 5] * n,
            '''labels''': [[1] * 5] * n,
            '''answers''': [{'''answer_start''': [9_7], '''text''': ['''1976''']}] * 1_0,
            '''id''': list(range(UpperCamelCase_	)	),
        } ,       features=UpperCamelCase_ ,       )
    return dataset



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       int	)->  Optional[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''file.arrow'''	)
    dataset.map(cache_file_name=UpperCamelCase_	)
    return filename


# FILE_CONTENT + files


_lowercase						  =		"\\n    Text data.\n    Second line of data."



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  List[Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt'''
    A__		 =					FILE_CONTENT
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_	)
    return filename



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  Optional[Any]:
    import bza

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.bz2'''
    A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
    with bza.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any]	)->  int:
    import gzip

    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''file.txt.gz'''	)
    A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
    with gzip.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  Any:
    if datasets.config.LZ4_AVAILABLE:
        import lza.frame

        A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.lz4'''
        A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
        with lza.frame.open(UpperCamelCase_ ,       '''wb'''	) as f:
            f.write(UpperCamelCase_	)
        return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[Any] ,       UpperCamelCase_				:       Tuple	)->  int:
    if datasets.config.PY7ZR_AVAILABLE:
        import pyazr

        A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.7z'''
        with pyazr.SevenZipFile(UpperCamelCase_ ,       '''w'''	) as archive:
            archive.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Tuple ,       UpperCamelCase_				:       int	)->  Optional[Any]:
    import tarfile

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.tar'''
    with tarfile.TarFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.add(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  str:
    import lzma

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.xz'''
    A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
    with lzma.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int] ,       UpperCamelCase_				:       List[str]	)->  List[str]:
    import zipfile

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Tuple	)->  str:
    if datasets.config.ZSTANDARD_AVAILABLE:
        import zstandard as zstd

        A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.zst'''
        A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
        with zstd.open(UpperCamelCase_ ,       '''wb'''	) as f:
            f.write(UpperCamelCase_	)
        return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  int:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.xml'''
    A__		 =					textwrap.dedent(
        '''\
    <?xml version="1.0" encoding="UTF-8" ?>
    <tmx version="1.4">
      <header segtype="sentence" srclang="ca" />
      <body>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang="en"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang="en"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang="en"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang="en"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang="en"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_	)
    return filename


_lowercase						  =		[
    {"col_1": "0", "col_2": 0, "col_3": 0.0},
    {"col_1": "1", "col_2": 1, "col_3": 1.0},
    {"col_1": "2", "col_2": 2, "col_3": 2.0},
    {"col_1": "3", "col_2": 3, "col_3": 3.0},
]
_lowercase						  =		[
    {"col_1": "4", "col_2": 4, "col_3": 4.0},
    {"col_1": "5", "col_2": 5, "col_3": 5.0},
]
_lowercase						  =		{
    "col_1": ["0", "1", "2", "3"],
    "col_2": [0, 1, 2, 3],
    "col_3": [0.0, 1.0, 2.0, 3.0],
}

_lowercase						  =		[
    {"col_3": 0.0, "col_1": "0", "col_2": 0},
    {"col_3": 1.0, "col_1": "1", "col_2": 1},
]

_lowercase						  =		[
    {"col_1": "s0", "col_2": 0, "col_3": 0.0},
    {"col_1": "s1", "col_2": 1, "col_3": 1.0},
    {"col_1": "s2", "col_2": 2, "col_3": 2.0},
    {"col_1": "s3", "col_2": 3, "col_3": 3.0},
]



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  str:
    return DATA_DICT_OF_LISTS



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[Any]	)->  List[str]:
    A__		 =					datasets.Dataset.from_dict(UpperCamelCase_	)
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.arrow'''	)
    dataset.map(cache_file_name=UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  List[str]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.sqlite'''	)
    with contextlib.closing(sqlitea.connect(UpperCamelCase_	)	) as con:
        A__		 =					con.cursor()
        cur.execute('''CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'''	)
        for item in DATA:
            cur.execute('''INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)''' ,       tuple(item.values()	)	)
        con.commit()
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any]	)->  Tuple:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.csv'''	)
    with open(UpperCamelCase_ ,       '''w''' ,       newline=''''''	) as f:
        A__		 =					csv.DictWriter(UpperCamelCase_ ,       fieldnames=['''col_1''', '''col_2''', '''col_3''']	)
        writer.writeheader()
        for item in DATA:
            writer.writerow(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  List[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset2.csv'''	)
    with open(UpperCamelCase_ ,       '''w''' ,       newline=''''''	) as f:
        A__		 =					csv.DictWriter(UpperCamelCase_ ,       fieldnames=['''col_1''', '''col_2''', '''col_3''']	)
        writer.writeheader()
        for item in DATA:
            writer.writerow(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       Optional[int]	)->  List[str]:
    import bza

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.csv.bz2'''
    with open(UpperCamelCase_ ,       '''rb'''	) as f:
        A__		 =					f.read()
    # data = bytes(FILE_CONTENT, "utf-8")
    with bza.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str ,       UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       Optional[int]	)->  str:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.csv.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       str ,       UpperCamelCase_				:       Any	)->  List[Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.csv.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(csv_path.replace('''.csv''' ,       '''.CSV'''	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(csva_path.replace('''.csv''' ,       '''.CSV'''	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str ,       UpperCamelCase_				:       str ,       UpperCamelCase_				:       Optional[Any]	)->  Tuple:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_with_dir.csv.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  Optional[int]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.parquet'''	)
    A__		 =					pa.schema(
        {
            '''col_1''': pa.string(),
            '''col_2''': pa.intaa(),
            '''col_3''': pa.floataa(),
        }	)
    with open(UpperCamelCase_ ,       '''wb'''	) as f:
        A__		 =					pq.ParquetWriter(UpperCamelCase_ ,       schema=UpperCamelCase_	)
        A__		 =					pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(UpperCamelCase_	)	)] for k in DATA[0]} ,       schema=UpperCamelCase_	)
        writer.write_table(UpperCamelCase_	)
        writer.close()
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  str:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.json'''	)
    A__		 =					{'''data''': DATA}
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        json.dump(UpperCamelCase_ ,       UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  Optional[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.json'''	)
    A__		 =					{'''data''': DATA_DICT_OF_LISTS}
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        json.dump(UpperCamelCase_ ,       UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  List[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  Tuple:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset2.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int	)->  Optional[int]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset_312.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA_312:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int	)->  Union[str, Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset-str.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA_STR:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any] ,       UpperCamelCase_				:       str	)->  List[str]:
    import gzip

    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.txt.gz'''	)
    with open(UpperCamelCase_ ,       '''rb'''	) as orig_file:
        with gzip.open(UpperCamelCase_ ,       '''wb'''	) as zipped_file:
            zipped_file.writelines(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Any ,       UpperCamelCase_				:       Optional[Any]	)->  List[Any]:
    import gzip

    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl.gz'''	)
    with open(UpperCamelCase_ ,       '''rb'''	) as orig_file:
        with gzip.open(UpperCamelCase_ ,       '''wb'''	) as zipped_file:
            zipped_file.writelines(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       List[str]	)->  Optional[int]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       Optional[int]	)->  Union[str, Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_nested.jsonl.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''nested''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       Any	)->  Any:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_with_dir.jsonl.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict ,       UpperCamelCase_				:       Any ,       UpperCamelCase_				:       str	)->  Tuple:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl.tar'''
    with tarfile.TarFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.add(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.add(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int] ,       UpperCamelCase_				:       int ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       List[str]	)->  str:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_nested.jsonl.tar'''
    with tarfile.TarFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.add(UpperCamelCase_ ,       arcname=os.path.join('''nested''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Any	)->  List[str]:
    A__		 =					['''0''', '''1''', '''2''', '''3''']
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.txt'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in data:
            f.write(item + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int	)->  Dict:
    A__		 =					['''0''', '''1''', '''2''', '''3''']
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset2.txt'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in data:
            f.write(item + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any]	)->  Union[str, Any]:
    A__		 =					['''0''', '''1''', '''2''', '''3''']
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.abc'''
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in data:
            f.write(item + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       Any ,       UpperCamelCase_				:       Optional[Any]	)->  List[str]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.text.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Any ,       UpperCamelCase_				:       List[Any] ,       UpperCamelCase_				:       Optional[int]	)->  Tuple:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_with_dir.text.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       Tuple ,       UpperCamelCase_				:       Tuple	)->  Any:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.ext.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename('''unsupported.ext'''	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename('''unsupported_2.ext'''	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  Any:
    A__		 =					'''\n'''.join(['''First''', '''Second\u2029with Unicode new line''', '''Third''']	)
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset_with_unicode_new_lines.txt'''	)
    with open(UpperCamelCase_ ,       '''w''' ,       encoding='''utf-8'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  Tuple:
    return os.path.join('''tests''' ,       '''features''' ,       '''data''' ,       '''test_image_rgb.jpg'''	)



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  Dict:
    return os.path.join('''tests''' ,       '''features''' ,       '''data''' ,       '''test_audio_44100.wav'''	)



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any] ,       UpperCamelCase_				:       Union[str, Any]	)->  List[Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.img.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	).replace('''.jpg''' ,       '''2.jpg'''	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any]	)->  str:
    A__		 =					tmp_path_factory.mktemp('''data_dir'''	)

    (data_dir / "subdir").mkdir()
    with open(data_dir / '''subdir''' / '''train.txt''' ,       '''w'''	) as f:
        f.write('''foo\n''' * 1_0	)
    with open(data_dir / '''subdir''' / '''test.txt''' ,       '''w'''	) as f:
        f.write('''bar\n''' * 1_0	)
    # hidden file
    with open(data_dir / '''subdir''' / '''.test.txt''' ,       '''w'''	) as f:
        f.write('''bar\n''' * 1_0	)

    # hidden directory
    (data_dir / ".subdir").mkdir()
    with open(data_dir / '''.subdir''' / '''train.txt''' ,       '''w'''	) as f:
        f.write('''foo\n''' * 1_0	)
    with open(data_dir / '''.subdir''' / '''test.txt''' ,       '''w'''	) as f:
        f.write('''bar\n''' * 1_0	)

    return data_dir
