

import unittest

from transformers.testing_utils import require_bsa
from transformers.utils import is_bsa_available

from ...test_feature_extraction_common import FeatureExtractionSavingTestMixin


if is_bsa_available():
      from transformers import MarkupLMFeatureExtractor

class   _a					(			unittest.TestCase				):




      """simple docstring"""

      def __init__(   self							:				Optional[int]						,			UpperCAmelCase							:				Union[str, Any]	):
            A_						  =				parent







      def 		__A	(   self							:				List[str]	):
            return {}




def       __snake_case						( ):



      """simple docstring"""

      A_						  =				"<HTML>\n\n    <HEAD>\n    <TITLE>sample document</TITLE>\n    </HEAD>\n\n    <BODY BGCOLOR=\"FFFFFF\">\n    <HR>\n    <a href=\"http://google.com\">Goog</a>\n    <H1>This is one header</H1>\n    <H2>This is a another Header</H2>\n    <P>Travel from\n        <P>\n        <B>SFO to JFK</B>\n        <BR>\n        <B><I>on May 2, 2015 at 2:00 pm. For details go to confirm.com </I></B>\n        <HR>\n        <div style=\"color:#0000FF\">\n            <h3>Traveler <b> name </b> is\n            <p> John Doe </p>\n        </div>"

      A_						  =				"\n    <!DOCTYPE html>\n    <html>\n    <body>\n\n    <h1>My First Heading</h1>\n    <p>My first paragraph.</p>\n\n    </body>\n    </html>\n    "

      return [html_string_a, html_string_a]



@require_bsa
class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Dict      		=       MarkupLMFeatureExtractor if is_bsa_available() else None





      def 		__A	(   self							:				int	):
            A_						  =				MarkupLMFeatureExtractionTester(self	)





      @property
      def 		__A	(   self							:				List[str]	):
            return self.feature_extract_tester.prepare_feat_extract_dict()







      def 		__A	(   self							:				Tuple	):
            # Initialize feature_extractor
            A_						  =				self.feature_extraction_class()

            # Test not batched input
            A_						  =				get_html_strings()[0]
            A_						  =				feature_extractor(UpperCAmelCase	)

            # fmt: off
            A_						  =				[["sample document", "Goog", "This is one header", "This is a another Header", "Travel from", "SFO to JFK", "on May 2, 2015 at 2:00 pm. For details go to confirm.com", "Traveler", "name", "is", "John Doe"]]
            A_						  =				[["/html/head/title", "/html/body/a", "/html/body/h1", "/html/body/h2", "/html/body/p", "/html/body/p/p/b[1]", "/html/body/p/p/b[2]/i", "/html/body/p/p/div/h3", "/html/body/p/p/div/h3/b", "/html/body/p/p/div/h3", "/html/body/p/p/div/h3/p"]]
            # fmt: on

            self.assertEqual(encoding.nodes						,			UpperCAmelCase	)
            self.assertEqual(encoding.xpaths						,			UpperCAmelCase	)

            # Test batched
            A_						  =				get_html_strings()
            A_						  =				feature_extractor(UpperCAmelCase	)

            # fmt: off
            A_						  =				expected_nodes + [["My First Heading", "My first paragraph."]]
            A_						  =				expected_xpaths + [["/html/body/h1", "/html/body/p"]]

            self.assertEqual(len(encoding.nodes	)						,			2	)
            self.assertEqual(len(encoding.xpaths	)						,			2	)

            self.assertEqual(encoding.nodes						,			UpperCAmelCase	)
            self.assertEqual(encoding.xpaths						,			UpperCAmelCase	)







import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

def   __lowercase       (			snake_case,	snake_case			):







				"""simple docstring"""






				assert isinstance(snake_case,	snake_case			)
				assert dataset.num_rows == 4
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_1", "col_2", "col_3"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('''keep_in_memory''',	[False, True]			)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:Tuple   =   tmp_path / '''cache'''
				__magic_name__		:int   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								__magic_name__		:Optional[Any]   =   ParquetDatasetReader(snake_case,	cache_dir=snake_case,	keep_in_memory=snake_case			).read()
				_check_parquet_dataset(snake_case,	snake_case			)

@pytest.mark.parametrize(
    '''features''',	[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ],	)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:List[str]   =   tmp_path / '''cache'''
				__magic_name__		:int   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				__magic_name__		:Tuple   =   features.copy() if features else default_expected_features
				__magic_name__		:Union[str, Any]   =   (
				    Features({feature: Value(snake_case			) for feature, dtype in features.items()}			) if features is not None else None
				)
				__magic_name__		:int   =   ParquetDatasetReader(snake_case,	features=snake_case,	cache_dir=snake_case			).read()
				_check_parquet_dataset(snake_case,	snake_case			)

@pytest.mark.parametrize('''split''',	[None, NamedSplit('''train'''			), '''train''', '''test''']			)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:str   =   tmp_path / '''cache'''
				__magic_name__		:List[Any]   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				__magic_name__		:int   =   ParquetDatasetReader(snake_case,	cache_dir=snake_case,	split=snake_case			).read()
				_check_parquet_dataset(snake_case,	snake_case			)
				assert dataset.split == split if split else "train"

@pytest.mark.parametrize('''path_type''',	[str, list]			)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				if issubclass(snake_case,	snake_case			):
								__magic_name__		:Union[str, Any]   =   parquet_path
				elif issubclass(snake_case,	snake_case			):
								__magic_name__		:Union[str, Any]   =   [parquet_path]
				__magic_name__		:Optional[int]   =   tmp_path / '''cache'''
				__magic_name__		:Optional[int]   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				__magic_name__		:str   =   ParquetDatasetReader(snake_case,	cache_dir=snake_case			).read()
				_check_parquet_dataset(snake_case,	snake_case			)

def   __lowercase       (			snake_case,	snake_case,	snake_case=("train",)			):







				"""simple docstring"""






				assert isinstance(snake_case,	snake_case			)
				for split in splits:
								__magic_name__		:Optional[Any]   =   dataset_dict[split]
								assert dataset.num_rows == 4
								assert dataset.num_columns == 3
								assert dataset.column_names == ["col_1", "col_2", "col_3"]
								for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('''keep_in_memory''',	[False, True]			)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:Any   =   tmp_path / '''cache'''
				__magic_name__		:Optional[int]   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								__magic_name__		:Tuple   =   ParquetDatasetReader(
								    {'''train''': parquet_path},	cache_dir=snake_case,	keep_in_memory=snake_case			).read()
				_check_parquet_datasetdict(snake_case,	snake_case			)

@pytest.mark.parametrize(
    '''features''',	[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ],	)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:Optional[Any]   =   tmp_path / '''cache'''
				__magic_name__		:Dict   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				__magic_name__		:int   =   features.copy() if features else default_expected_features
				__magic_name__		:List[Any]   =   (
				    Features({feature: Value(snake_case			) for feature, dtype in features.items()}			) if features is not None else None
				)
				__magic_name__		:Optional[int]   =   ParquetDatasetReader({'''train''': parquet_path},	features=snake_case,	cache_dir=snake_case			).read()
				_check_parquet_datasetdict(snake_case,	snake_case			)

@pytest.mark.parametrize('''split''',	[None, NamedSplit('''train'''			), '''train''', '''test''']			)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				if split:
								__magic_name__		:Dict   =   {split: parquet_path}
				else:
								__magic_name__		:Optional[int]   =   '''train'''
								__magic_name__		:Dict   =   {'''train''': parquet_path, '''test''': parquet_path}
				__magic_name__		:List[Any]   =   tmp_path / '''cache'''
				__magic_name__		:Optional[Any]   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				__magic_name__		:Optional[Any]   =   ParquetDatasetReader(snake_case,	cache_dir=snake_case			).read()
				_check_parquet_datasetdict(snake_case,	snake_case,	splits=list(path.keys()			)			)
				assert all(dataset[split].split == split for split in path.keys()			)

def   __lowercase       (			snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:str   =   ParquetDatasetWriter(snake_case,	tmp_path / '''foo.parquet'''			)
				assert writer.write() > 0
				__magic_name__		:List[Any]   =   pq.ParquetFile(tmp_path / '''foo.parquet'''			)
				__magic_name__		:List[Any]   =   pf.read()
				assert dataset.data.table == output_table

def   __lowercase       (			snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:List[str]   =   str(shared_datadir / '''test_image_rgb.jpg'''			)
				__magic_name__		:Tuple   =   {'''image''': [image_path]}
				__magic_name__		:List[Any]   =   Features({'''image''': Image()}			)
				__magic_name__		:Tuple   =   Dataset.from_dict(snake_case,	features=snake_case			)
				__magic_name__		:Union[str, Any]   =   ParquetDatasetWriter(snake_case,	tmp_path / '''foo.parquet'''			)
				assert writer.write() > 0

				__magic_name__		:List[str]   =   Dataset.from_parquet(str(tmp_path / '''foo.parquet'''			)			)
				assert dataset.features == reloaded_dataset.features

				__magic_name__		:List[str]   =   ParquetDatasetReader(str(tmp_path / '''foo.parquet'''			),	streaming=snake_case			).read()
				assert dataset.features == reloaded_iterable_dataset.features

@pytest.mark.parametrize(
    '''feature, expected''',	[
        (Features({'''foo''': Value('''int32'''			)}			), None),
        (Features({'''image''': Image(), '''foo''': Value('''int32'''			)}			), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({'''nested''': Sequence(Audio()			)}			), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ],	)
def   __lowercase       (			snake_case,	snake_case			):







				"""simple docstring"""






				assert get_writer_batch_size(snake_case			) == expected






