





"""simple docstring"""
import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    Dict  ,     _UpperCAmelCase					:    Dict			):
 assert isinstance(_lowerCamelCase  ,     _lowerCamelCase			)
 assert dataset.num_rows == 4
 assert dataset.num_columns == 3
 assert dataset.column_names == ["col_1", "col_2", "col_3"]
 for feature, expected_dtype in expected_features.items():
  assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('keep_in_memory'  ,     [False, True]			)
def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    List[Any]  ,     _UpperCAmelCase					:    Dict  ,     _UpperCAmelCase					:    List[str]			):
 lowerCAmelCase    				= tmp_path / 'cache'
 lowerCAmelCase    				= {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
 with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
  lowerCAmelCase    				= JsonDatasetReader(_lowerCamelCase  ,     cache_dir=_lowerCamelCase  ,     keep_in_memory=_lowerCamelCase			).read()
 _check_json_dataset(_lowerCamelCase  ,     _lowerCamelCase			)

@pytest.mark.parametrize(
    'features'  ,     [
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]  ,     )
def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    Dict  ,     _UpperCAmelCase					:    Tuple  ,     _UpperCAmelCase					:    str			):
 lowerCAmelCase    				= tmp_path / 'cache'
 lowerCAmelCase    				= {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
 lowerCAmelCase    				= features.copy() if features else default_expected_features
 lowerCAmelCase    				= (
     Features({feature: Value(_lowerCamelCase			) for feature, dtype in features.items()}			) if features is not None else None
 )
 lowerCAmelCase    				= JsonDatasetReader(_lowerCamelCase  ,     features=_lowerCamelCase  ,     cache_dir=_lowerCamelCase			).read()
 _check_json_dataset(_lowerCamelCase  ,     _lowerCamelCase			)

@pytest.mark.parametrize(
    'features'  ,     [
        None,
        {'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'},
    ]  ,     )
def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    int  ,     _UpperCAmelCase					:    Tuple  ,     _UpperCAmelCase					:    Tuple			):
 lowerCAmelCase    				= tmp_path / 'cache'
 lowerCAmelCase    				= {'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'}
 lowerCAmelCase    				= features.copy() if features else default_expected_features
 lowerCAmelCase    				= (
     Features({feature: Value(_lowerCamelCase			) for feature, dtype in features.items()}			) if features is not None else None
 )
 lowerCAmelCase    				= JsonDatasetReader(_lowerCamelCase  ,     features=_lowerCamelCase  ,     cache_dir=_lowerCamelCase			).read()
 assert isinstance(_lowerCamelCase  ,     _lowerCamelCase			)
 assert dataset.num_rows == 2
 assert dataset.num_columns == 3
 assert dataset.column_names == ["col_3", "col_1", "col_2"]
 for feature, expected_dtype in expected_features.items():
  assert dataset.features[feature].dtype == expected_dtype

def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    Optional[int]  ,     _UpperCAmelCase					:    List[str]			):
 lowerCAmelCase    				= {'col_2': 'int64', 'col_3': 'float64', 'col_1': 'string'}
 lowerCAmelCase    				= features.copy()
 lowerCAmelCase    				= (
     Features({feature: Value(_lowerCamelCase			) for feature, dtype in features.items()}			) if features is not None else None
 )
 lowerCAmelCase    				= tmp_path / 'cache'
 lowerCAmelCase    				= JsonDatasetReader(_lowerCamelCase  ,     features=_lowerCamelCase  ,     cache_dir=_lowerCamelCase			).read()
 assert isinstance(_lowerCamelCase  ,     _lowerCamelCase			)
 assert dataset.num_rows == 2
 assert dataset.num_columns == 3
 assert dataset.column_names == ["col_2", "col_3", "col_1"]
 for feature, expected_dtype in expected_features.items():
  assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('split'  ,     [None, NamedSplit('train'			), 'train', 'test']			)
def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    str  ,     _UpperCAmelCase					:    int  ,     _UpperCAmelCase					:    Any			):
 lowerCAmelCase    				= tmp_path / 'cache'
 lowerCAmelCase    				= {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
 lowerCAmelCase    				= JsonDatasetReader(_lowerCamelCase  ,     cache_dir=_lowerCamelCase  ,     split=_lowerCamelCase			).read()
 _check_json_dataset(_lowerCamelCase  ,     _lowerCamelCase			)
 assert dataset.split == split if split else "train"

@pytest.mark.parametrize('path_type'  ,     [str, list]			)
def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    int  ,     _UpperCAmelCase					:    Any  ,     _UpperCAmelCase					:    Tuple			):




 if issubclass(_lowerCamelCase  ,     _lowerCamelCase			):
  lowerCAmelCase    				= jsonl_path
 elif issubclass(_lowerCamelCase  ,     _lowerCamelCase			):
  lowerCAmelCase    				= [jsonl_path]
 lowerCAmelCase    				= tmp_path / 'cache'
 lowerCAmelCase    				= {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
 lowerCAmelCase    				= JsonDatasetReader(_lowerCamelCase  ,     cache_dir=_lowerCamelCase			).read()
 _check_json_dataset(_lowerCamelCase  ,     _lowerCamelCase			)

def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    str  ,     _UpperCAmelCase					:    Union[str, Any]  ,     _UpperCAmelCase					:    Optional[int]=("train",)			):
 assert isinstance(_lowerCamelCase  ,     _lowerCamelCase			)
 for split in splits:
  lowerCAmelCase    				= dataset_dict[split]
  assert dataset.num_rows == 4
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_1", "col_2", "col_3"]
  for feature, expected_dtype in expected_features.items():
   assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('keep_in_memory'  ,     [False, True]			)
def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    Tuple  ,     _UpperCAmelCase					:    int  ,     _UpperCAmelCase					:    List[str]			):
 lowerCAmelCase    				= tmp_path / 'cache'
 lowerCAmelCase    				= {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
 with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
  lowerCAmelCase    				= JsonDatasetReader({'train': jsonl_path}  ,     cache_dir=_lowerCamelCase  ,     keep_in_memory=_lowerCamelCase			).read()
 _check_json_datasetdict(_lowerCamelCase  ,     _lowerCamelCase			)

@pytest.mark.parametrize(
    'features'  ,     [
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]  ,     )
def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    Dict  ,     _UpperCAmelCase					:    Tuple  ,     _UpperCAmelCase					:    Any			):
 lowerCAmelCase    				= tmp_path / 'cache'
 lowerCAmelCase    				= {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
 lowerCAmelCase    				= features.copy() if features else default_expected_features
 lowerCAmelCase    				= (
     Features({feature: Value(_lowerCamelCase			) for feature, dtype in features.items()}			) if features is not None else None
 )
 lowerCAmelCase    				= JsonDatasetReader({'train': jsonl_path}  ,     features=_lowerCamelCase  ,     cache_dir=_lowerCamelCase			).read()
 _check_json_datasetdict(_lowerCamelCase  ,     _lowerCamelCase			)

@pytest.mark.parametrize('split'  ,     [None, NamedSplit('train'			), 'train', 'test']			)
def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    str  ,     _UpperCAmelCase					:    str  ,     _UpperCAmelCase					:    str			):




 if split:
  lowerCAmelCase    				= {split: jsonl_path}
 else:
  lowerCAmelCase    				= 'train'
  lowerCAmelCase    				= {'train': jsonl_path, 'test': jsonl_path}
 lowerCAmelCase    				= tmp_path / 'cache'
 lowerCAmelCase    				= {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
 lowerCAmelCase    				= JsonDatasetReader(_lowerCamelCase  ,     cache_dir=_lowerCamelCase			).read()
 _check_json_datasetdict(_lowerCamelCase  ,     _lowerCamelCase  ,     splits=list(path.keys()			)			)
 assert all(dataset[split].split == split for split in path.keys()			)

def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    Optional[Any]			):
 return json.load(_lowerCamelCase			)

def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    Union[str, Any]			):
 return [json.loads(_lowerCamelCase			) for line in buffer]







class       a							:



 @pytest.mark.parametrize('lines, load_json_function'				,       [(True, load_json_lines), (False, load_json)]     )
 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case				,       _snake_case     ):







  """simple docstring"""






  with io.BytesIO() as buffer:
   JsonDatasetWriter(SCREAMING_SNAKE_CASE_				,       SCREAMING_SNAKE_CASE_				,       lines=SCREAMING_SNAKE_CASE_     ).write()
   buffer.seek(0     )
   lowerCAmelCase    				= load_json_function(SCREAMING_SNAKE_CASE_     )
  assert isinstance(SCREAMING_SNAKE_CASE_				,       SCREAMING_SNAKE_CASE_     )
  assert isinstance(exported_content[0]				,       SCREAMING_SNAKE_CASE_     )
  assert len(SCREAMING_SNAKE_CASE_     ) == 10



 @pytest.mark.parametrize(
     'orient, container, keys, len_at'				,       [
         ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
         ('split', dict, {'columns', 'data'}, 'data'),
         ('index', dict, set('0123456789'     ), None),
         ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
         ('values', list, None, None),
         ('table', dict, {'schema', 'data'}, 'data'),
     ]				,       )
 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case     ):







  """simple docstring"""






  with io.BytesIO() as buffer:
   JsonDatasetWriter(SCREAMING_SNAKE_CASE_				,       SCREAMING_SNAKE_CASE_				,       lines=SCREAMING_SNAKE_CASE_				,       orient=SCREAMING_SNAKE_CASE_     ).write()
   buffer.seek(0     )
   lowerCAmelCase    				= load_json(SCREAMING_SNAKE_CASE_     )
  assert isinstance(SCREAMING_SNAKE_CASE_				,       SCREAMING_SNAKE_CASE_     )
  if keys:
   if container is dict:
    assert exported_content.keys() == keys
   else:
    assert exported_content[0].keys() == keys
  else:
   assert not hasattr(SCREAMING_SNAKE_CASE_				,       'keys'     ) and not hasattr(exported_content[0]				,       'keys'     )
  if len_at:
   assert len(exported_content[len_at]     ) == 10
  else:
   assert len(SCREAMING_SNAKE_CASE_     ) == 10



 @pytest.mark.parametrize('lines, load_json_function'				,       [(True, load_json_lines), (False, load_json)]     )
 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case				,       _snake_case     ):







  """simple docstring"""






  with io.BytesIO() as buffer:
   JsonDatasetWriter(SCREAMING_SNAKE_CASE_				,       SCREAMING_SNAKE_CASE_				,       lines=SCREAMING_SNAKE_CASE_				,       num_proc=2     ).write()
   buffer.seek(0     )
   lowerCAmelCase    				= load_json_function(SCREAMING_SNAKE_CASE_     )
  assert isinstance(SCREAMING_SNAKE_CASE_				,       SCREAMING_SNAKE_CASE_     )
  assert isinstance(exported_content[0]				,       SCREAMING_SNAKE_CASE_     )
  assert len(SCREAMING_SNAKE_CASE_     ) == 10



 @pytest.mark.parametrize(
     'orient, container, keys, len_at'				,       [
         ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
         ('split', dict, {'columns', 'data'}, 'data'),
         ('index', dict, set('0123456789'     ), None),
         ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
         ('values', list, None, None),
         ('table', dict, {'schema', 'data'}, 'data'),
     ]				,       )
 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case     ):







  """simple docstring"""






  with io.BytesIO() as buffer:
   JsonDatasetWriter(SCREAMING_SNAKE_CASE_				,       SCREAMING_SNAKE_CASE_				,       lines=SCREAMING_SNAKE_CASE_				,       orient=SCREAMING_SNAKE_CASE_				,       num_proc=2     ).write()
   buffer.seek(0     )
   lowerCAmelCase    				= load_json(SCREAMING_SNAKE_CASE_     )
  assert isinstance(SCREAMING_SNAKE_CASE_				,       SCREAMING_SNAKE_CASE_     )
  if keys:
   if container is dict:
    assert exported_content.keys() == keys
   else:
    assert exported_content[0].keys() == keys
  else:
   assert not hasattr(SCREAMING_SNAKE_CASE_				,       'keys'     ) and not hasattr(exported_content[0]				,       'keys'     )
  if len_at:
   assert len(exported_content[len_at]     ) == 10
  else:
   assert len(SCREAMING_SNAKE_CASE_     ) == 10



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  with pytest.raises(SCREAMING_SNAKE_CASE_     ):
   with io.BytesIO() as buffer:
    JsonDatasetWriter(SCREAMING_SNAKE_CASE_				,       SCREAMING_SNAKE_CASE_				,       num_proc=0     )



 @pytest.mark.parametrize('compression, extension'				,       [('gzip', 'gz'), ('bz2', 'bz2'), ('xz', 'xz')]     )
 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= tmp_path_factory.mktemp('data'     ) / F'test.json.{extension}'
  lowerCAmelCase    				= str(shared_datadir / F'test_file.json.{extension}'     )
  JsonDatasetWriter(SCREAMING_SNAKE_CASE_				,       SCREAMING_SNAKE_CASE_				,       compression=SCREAMING_SNAKE_CASE_     ).write()

  with fsspec.open(SCREAMING_SNAKE_CASE_				,       'rb'				,       compression='infer'     ) as f:
   lowerCAmelCase    				= f.read()
  with fsspec.open(SCREAMING_SNAKE_CASE_				,       'rb'				,       compression='infer'     ) as f:
   lowerCAmelCase    				= f.read()
  assert exported_content == original_content







"""simple docstring"""






from typing import List, Optional, Union

from ...image_utils import ImageInput
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType



class 			lowercase					( lowercase__		):
			lowercase        =     ['''image_processor''', '''tokenizer''']
			lowercase        =     '''BlipImageProcessor'''
			lowercase        =     ('''BertTokenizer''', '''BertTokenizerFast''')






			def __init__(self  :	Any      ,SCREAMING_SNAKE_CASE_  :	List[str]      ,SCREAMING_SNAKE_CASE_  :	Optional[int]			)       ->	Dict:





				"""simple docstring"""


				lowerCAmelCase												=  False
				super().__init__(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)
				lowerCAmelCase												=  self.image_processor






			def __call__(self  :	int      ,SCREAMING_SNAKE_CASE_  :	ImageInput = None      ,SCREAMING_SNAKE_CASE_  :	Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None      ,SCREAMING_SNAKE_CASE_  :	bool = True      ,SCREAMING_SNAKE_CASE_  :	Union[bool, str, PaddingStrategy] = False      ,SCREAMING_SNAKE_CASE_  :	Union[bool, str, TruncationStrategy] = None      ,SCREAMING_SNAKE_CASE_  :	Optional[int] = None      ,SCREAMING_SNAKE_CASE_  :	int = 0      ,SCREAMING_SNAKE_CASE_  :	Optional[int] = None      ,SCREAMING_SNAKE_CASE_  :	Optional[bool] = None      ,SCREAMING_SNAKE_CASE_  :	bool = False      ,SCREAMING_SNAKE_CASE_  :	bool = False      ,SCREAMING_SNAKE_CASE_  :	bool = False      ,SCREAMING_SNAKE_CASE_  :	bool = False      ,SCREAMING_SNAKE_CASE_  :	bool = False      ,SCREAMING_SNAKE_CASE_  :	bool = True      ,SCREAMING_SNAKE_CASE_  :	Optional[Union[str, TensorType]] = None      ,**SCREAMING_SNAKE_CASE_  :	str      ,)       ->	BatchEncoding:





				"""simple docstring"""


				if images is None and text is None:
					raise ValueError('''You have to specify either images or text.'''			)

				# Get only text
				if images is None:
					lowerCAmelCase												=  self.tokenizer
					lowerCAmelCase												=  self.tokenizer(
					    text=SCREAMING_SNAKE_CASE_      ,add_special_tokens=SCREAMING_SNAKE_CASE_      ,padding=SCREAMING_SNAKE_CASE_      ,truncation=SCREAMING_SNAKE_CASE_      ,max_length=SCREAMING_SNAKE_CASE_      ,stride=SCREAMING_SNAKE_CASE_      ,pad_to_multiple_of=SCREAMING_SNAKE_CASE_      ,return_attention_mask=SCREAMING_SNAKE_CASE_      ,return_overflowing_tokens=SCREAMING_SNAKE_CASE_      ,return_special_tokens_mask=SCREAMING_SNAKE_CASE_      ,return_offsets_mapping=SCREAMING_SNAKE_CASE_      ,return_token_type_ids=SCREAMING_SNAKE_CASE_      ,return_length=SCREAMING_SNAKE_CASE_      ,verbose=SCREAMING_SNAKE_CASE_      ,return_tensors=SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_      ,)
					return text_encoding

				# add pixel_values
				lowerCAmelCase												=  self.image_processor(SCREAMING_SNAKE_CASE_      ,return_tensors=SCREAMING_SNAKE_CASE_			)

				if text is not None:
					lowerCAmelCase												=  self.tokenizer(
					    text=SCREAMING_SNAKE_CASE_      ,add_special_tokens=SCREAMING_SNAKE_CASE_      ,padding=SCREAMING_SNAKE_CASE_      ,truncation=SCREAMING_SNAKE_CASE_      ,max_length=SCREAMING_SNAKE_CASE_      ,stride=SCREAMING_SNAKE_CASE_      ,pad_to_multiple_of=SCREAMING_SNAKE_CASE_      ,return_attention_mask=SCREAMING_SNAKE_CASE_      ,return_overflowing_tokens=SCREAMING_SNAKE_CASE_      ,return_special_tokens_mask=SCREAMING_SNAKE_CASE_      ,return_offsets_mapping=SCREAMING_SNAKE_CASE_      ,return_token_type_ids=SCREAMING_SNAKE_CASE_      ,return_length=SCREAMING_SNAKE_CASE_      ,verbose=SCREAMING_SNAKE_CASE_      ,return_tensors=SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_      ,)
				else:
					lowerCAmelCase												=  None

				if text_encoding is not None:
					encoding_image_processor.update(SCREAMING_SNAKE_CASE_			)

				return encoding_image_processor






			def 					UpperCAmelCase							(self  :	List[str]      ,*SCREAMING_SNAKE_CASE_  :	str      ,**SCREAMING_SNAKE_CASE_  :	str			)       ->	Union[str, Any]:





				"""simple docstring"""


				return self.tokenizer.batch_decode(*SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_			)






			def 					UpperCAmelCase							(self  :	List[Any]      ,*SCREAMING_SNAKE_CASE_  :	str      ,**SCREAMING_SNAKE_CASE_  :	Optional[int]			)       ->	Any:





				"""simple docstring"""


				return self.tokenizer.decode(*SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_			)






			@property
			def 					UpperCAmelCase							(self  :	Dict			)       ->	Tuple:





				"""simple docstring"""


				lowerCAmelCase												=  self.tokenizer.model_input_names
				lowerCAmelCase												=  self.image_processor.model_input_names
				return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names			)			)


