

import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases





def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__   ):







							'''simple docstring'''






							assert isinstance(UpperCamelCase__     ,							UpperCamelCase__   )
							assert dataset.num_rows == 4
							assert dataset.num_columns == 3
							assert dataset.column_names == ["col_1", "col_2", "col_3"]
							for feature, expected_dtype in expected_features.items():
														assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('keep_in_memory'     ,							[False, True]   )
def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__   ):







							'''simple docstring'''






							snake_case_           =					tmp_path / 'cache'
							snake_case_           =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
							with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
														snake_case_           =					ParquetDatasetReader(UpperCamelCase__     ,							cache_dir=UpperCamelCase__     ,							keep_in_memory=UpperCamelCase__   ).read()
							_check_parquet_dataset(UpperCamelCase__     ,							UpperCamelCase__   )





@pytest.mark.parametrize(
    'features'     ,							[
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]     ,							)
def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__   ):







							'''simple docstring'''






							snake_case_           =					tmp_path / 'cache'
							snake_case_           =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
							snake_case_           =					features.copy() if features else default_expected_features
							snake_case_           =					(
							    Features({feature: Value(UpperCamelCase__   ) for feature, dtype in features.items()}   ) if features is not None else None
							)
							snake_case_           =					ParquetDatasetReader(UpperCamelCase__     ,							features=UpperCamelCase__     ,							cache_dir=UpperCamelCase__   ).read()
							_check_parquet_dataset(UpperCamelCase__     ,							UpperCamelCase__   )





@pytest.mark.parametrize('split'     ,							[None, NamedSplit('train'   ), 'train', 'test']   )
def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__   ):







							'''simple docstring'''






							snake_case_           =					tmp_path / 'cache'
							snake_case_           =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
							snake_case_           =					ParquetDatasetReader(UpperCamelCase__     ,							cache_dir=UpperCamelCase__     ,							split=UpperCamelCase__   ).read()
							_check_parquet_dataset(UpperCamelCase__     ,							UpperCamelCase__   )
							assert dataset.split == split if split else "train"





@pytest.mark.parametrize('path_type'     ,							[str, list]   )
def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__   ):







							'''simple docstring'''






							if issubclass(UpperCamelCase__     ,							UpperCamelCase__   ):
														snake_case_           =					parquet_path
							elif issubclass(UpperCamelCase__     ,							UpperCamelCase__   ):
														snake_case_           =					[parquet_path]
							snake_case_           =					tmp_path / 'cache'
							snake_case_           =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
							snake_case_           =					ParquetDatasetReader(UpperCamelCase__     ,							cache_dir=UpperCamelCase__   ).read()
							_check_parquet_dataset(UpperCamelCase__     ,							UpperCamelCase__   )





def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__=("train",)   ):







							'''simple docstring'''






							assert isinstance(UpperCamelCase__     ,							UpperCamelCase__   )
							for split in splits:
														snake_case_           =					dataset_dict[split]
														assert dataset.num_rows == 4
														assert dataset.num_columns == 3
														assert dataset.column_names == ["col_1", "col_2", "col_3"]
														for feature, expected_dtype in expected_features.items():
																					assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('keep_in_memory'     ,							[False, True]   )
def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__   ):







							'''simple docstring'''






							snake_case_           =					tmp_path / 'cache'
							snake_case_           =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
							with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
														snake_case_           =					ParquetDatasetReader(
														    {'train': parquet_path}     ,							cache_dir=UpperCamelCase__     ,							keep_in_memory=UpperCamelCase__   ).read()
							_check_parquet_datasetdict(UpperCamelCase__     ,							UpperCamelCase__   )





@pytest.mark.parametrize(
    'features'     ,							[
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]     ,							)
def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__   ):







							'''simple docstring'''






							snake_case_           =					tmp_path / 'cache'
							snake_case_           =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
							snake_case_           =					features.copy() if features else default_expected_features
							snake_case_           =					(
							    Features({feature: Value(UpperCamelCase__   ) for feature, dtype in features.items()}   ) if features is not None else None
							)
							snake_case_           =					ParquetDatasetReader({'train': parquet_path}     ,							features=UpperCamelCase__     ,							cache_dir=UpperCamelCase__   ).read()
							_check_parquet_datasetdict(UpperCamelCase__     ,							UpperCamelCase__   )





@pytest.mark.parametrize('split'     ,							[None, NamedSplit('train'   ), 'train', 'test']   )
def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__   ):







							'''simple docstring'''






							if split:
														snake_case_           =					{split: parquet_path}
							else:
														snake_case_           =					'train'
														snake_case_           =					{'train': parquet_path, 'test': parquet_path}
							snake_case_           =					tmp_path / 'cache'
							snake_case_           =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
							snake_case_           =					ParquetDatasetReader(UpperCamelCase__     ,							cache_dir=UpperCamelCase__   ).read()
							_check_parquet_datasetdict(UpperCamelCase__     ,							UpperCamelCase__     ,							splits=list(path.keys()   )   )
							assert all(dataset[split].split == split for split in path.keys()   )





def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__   ):







							'''simple docstring'''






							snake_case_           =					ParquetDatasetWriter(UpperCamelCase__     ,							tmp_path / 'foo.parquet'   )
							assert writer.write() > 0
							snake_case_           =					pq.ParquetFile(tmp_path / 'foo.parquet'   )
							snake_case_           =					pf.read()
							assert dataset.data.table == output_table





def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__   ):







							'''simple docstring'''






							snake_case_           =					str(shared_datadir / 'test_image_rgb.jpg'   )
							snake_case_           =					{'image': [image_path]}
							snake_case_           =					Features({'image': Image()}   )
							snake_case_           =					Dataset.from_dict(UpperCamelCase__     ,							features=UpperCamelCase__   )
							snake_case_           =					ParquetDatasetWriter(UpperCamelCase__     ,							tmp_path / 'foo.parquet'   )
							assert writer.write() > 0

							snake_case_           =					Dataset.from_parquet(str(tmp_path / 'foo.parquet'   )   )
							assert dataset.features == reloaded_dataset.features

							snake_case_           =					ParquetDatasetReader(str(tmp_path / 'foo.parquet'   )     ,							streaming=UpperCamelCase__   ).read()
							assert dataset.features == reloaded_iterable_dataset.features





@pytest.mark.parametrize(
    'feature, expected'     ,							[
        (Features({'foo': Value('int32'   )}   ), None),
        (Features({'image': Image(), 'foo': Value('int32'   )}   ), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({'nested': Sequence(Audio()   )}   ), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]     ,							)
def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__   ):







							'''simple docstring'''






							assert get_writer_batch_size(UpperCamelCase__   ) == expected







import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin







class 				lowercase       ( lowercase_			,						unittest.TestCase    ):
				__SCREAMING_SNAKE_CASE      :				int     =    PhobertTokenizer
				__SCREAMING_SNAKE_CASE      :				int     =    False




				def 					a							(      self ):
											super().setUp()

											# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
											snake_case_           =					['T@@', 'i', 'I', 'R@@', 'r', 'e@@']
											snake_case_           =					dict(zip(snake_case   ,     range(len(snake_case ) ) ) )
											snake_case_           =					['#version: 0.2', 'l à</w>']
											snake_case_           =					{'unk_token': '<unk>'}

											snake_case_           =					os.path.join(self.tmpdirname   ,     VOCAB_FILES_NAMES['vocab_file'] )
											snake_case_           =					os.path.join(self.tmpdirname   ,     VOCAB_FILES_NAMES['merges_file'] )

											with open(self.vocab_file   ,     'w'   ,     encoding='utf-8' ) as fp:
																		for token in vocab_tokens:
																									fp.write(F'''{token} {vocab_tokens[token]}\n''' )
											with open(self.merges_file   ,     'w'   ,     encoding='utf-8' ) as fp:
																		fp.write('\n'.join(snake_case ) )




				def 					a							(      self   ,     **snake_case ):
											kwargs.update(self.special_tokens_map )
											return PhobertTokenizer.from_pretrained(self.tmpdirname   ,     **snake_case )




				def 					a							(      self   ,     snake_case ):
											snake_case_           =					'Tôi là VinAI Research'
											snake_case_           =					'T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'
											return input_text, output_text




				def 					a							(      self ):
											snake_case_           =					PhobertTokenizer(self.vocab_file   ,     self.merges_file   ,     **self.special_tokens_map )
											snake_case_           =					'Tôi là VinAI Research'
											snake_case_           =					'T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'.split()
											snake_case_           =					tokenizer.tokenize(snake_case )
											print(snake_case )
											self.assertListEqual(snake_case   ,     snake_case )

											snake_case_           =					tokens + [tokenizer.unk_token]

											snake_case_           =					[4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
											self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case )   ,     snake_case )




