



"""simple docstring"""






import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






class 			__snake_case			(      _lowercase):




  def __init__(					self	:		List[Any]		,   __lowerCAmelCase	:		NestedDataStructureLike[PathLike]		,   __lowerCAmelCase	:		Optional[NamedSplit] = None		,   __lowerCAmelCase	:		Optional[Features] = None		,   __lowerCAmelCase	:		str = None		,   __lowerCAmelCase	:		bool = False		,   __lowerCAmelCase	:		bool = False		,   __lowerCAmelCase	:		Optional[str] = None		,   __lowerCAmelCase	:		Optional[int] = None		,   **__lowerCAmelCase	:		Optional[Any]		,   ):






         """simple docstring"""






         super().__init__(
             __lowerCAmelCase		,   split=__lowerCAmelCase		,   features=__lowerCAmelCase		,   cache_dir=__lowerCAmelCase		,   keep_in_memory=__lowerCAmelCase		,   streaming=__lowerCAmelCase		,   num_proc=__lowerCAmelCase		,   **__lowerCAmelCase		,   )
         _lowerCamelCase		:							Any            =					field
         _lowerCamelCase		:							Any            =					path_or_paths if isinstance(__lowerCAmelCase		,   __lowerCAmelCase						) else {self.split: path_or_paths}
         _lowerCamelCase		:							Optional[int]            =					Json(
             cache_dir=__lowerCAmelCase		,   data_files=__lowerCAmelCase		,   features=__lowerCAmelCase		,   field=__lowerCAmelCase		,   **__lowerCAmelCase		,   )




  def 		SCREAMING_SNAKE_CASE							(					self	:		int						):






         """simple docstring"""






         if self.streaming:
                _lowerCamelCase		:							Union[str, Any]            =					self.builder.as_streaming_dataset(split=self.split						)
         # Build regular (map-style) dataset
         else:
                _lowerCamelCase		:							List[Any]            =					None
                _lowerCamelCase		:							Any            =					None
                _lowerCamelCase		:							Union[str, Any]            =					None
                _lowerCamelCase		:							List[str]            =					None

                self.builder.download_and_prepare(
                    download_config=__lowerCAmelCase		,   download_mode=__lowerCAmelCase		,   verification_mode=__lowerCAmelCase		,   base_path=__lowerCAmelCase		,   num_proc=self.num_proc		,   )
                _lowerCamelCase		:							List[Any]            =					self.builder.as_dataset(
                    split=self.split		,   verification_mode=__lowerCAmelCase		,   in_memory=self.keep_in_memory						)
         return dataset






class 			__snake_case			:




  def __init__(					self	:		Optional[int]		,   __lowerCAmelCase	:		Dataset		,   __lowerCAmelCase	:		Union[PathLike, BinaryIO]		,   __lowerCAmelCase	:		Optional[int] = None		,   __lowerCAmelCase	:		Optional[int] = None		,   **__lowerCAmelCase	:		Dict		,   ):






         """simple docstring"""






         if num_proc is not None and num_proc <= 0:
                raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''						)

         _lowerCamelCase		:							Any            =					dataset
         _lowerCamelCase		:							Any            =					path_or_buf
         _lowerCamelCase		:							Any            =					batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
         _lowerCamelCase		:							str            =					num_proc
         _lowerCamelCase		:							Optional[int]            =					'''utf-8'''
         _lowerCamelCase		:							Optional[Any]            =					to_json_kwargs




  def 		SCREAMING_SNAKE_CASE							(					self	:		List[str]						):






         """simple docstring"""






         _lowerCamelCase		:							int            =					self.to_json_kwargs.pop('''path_or_buf'''		,   __lowerCAmelCase						)
         _lowerCamelCase		:							List[str]            =					self.to_json_kwargs.pop('''orient'''		,   '''records'''						)
         _lowerCamelCase		:							str            =					self.to_json_kwargs.pop('''lines'''		,   True if orient == '''records''' else False						)
         _lowerCamelCase		:							Tuple            =					self.to_json_kwargs.pop('''index'''		,   False if orient in ['''split''', '''table'''] else True						)
         _lowerCamelCase		:							Dict            =					self.to_json_kwargs.pop('''compression'''		,   __lowerCAmelCase						)

         if compression not in [None, "infer", "gzip", "bz2", "xz"]:
                raise NotImplementedError(f'''`datasets` currently does not support {compression} compression'''						)

         if isinstance(self.path_or_buf		,   (str, bytes, os.PathLike)						):
                with fsspec.open(self.path_or_buf		,   '''wb'''		,   compression=__lowerCAmelCase						) as buffer:
                       _lowerCamelCase		:							List[str]            =					self._write(file_obj=__lowerCAmelCase		,   orient=__lowerCAmelCase		,   lines=__lowerCAmelCase		,   index=__lowerCAmelCase		,   **self.to_json_kwargs						)
         else:
                if compression:
                       raise NotImplementedError(
                           f'''The compression parameter is not supported when writing to a buffer, but compression={compression}'''
                           ''' was passed. Please provide a local path instead.'''						)
                _lowerCamelCase		:							List[str]            =					self._write(
                    file_obj=self.path_or_buf		,   orient=__lowerCAmelCase		,   lines=__lowerCAmelCase		,   index=__lowerCAmelCase		,   **self.to_json_kwargs						)
         return written




  def 		SCREAMING_SNAKE_CASE							(					self	:		Dict		,   __lowerCAmelCase	:		Optional[int]						):






         """simple docstring"""






         _lowerCamelCase     ,       _lowerCamelCase     ,       _lowerCamelCase     ,       _lowerCamelCase     ,       _lowerCamelCase		:							str            =					args

         _lowerCamelCase		:							Any            =					query_table(
             table=self.dataset.data		,   key=slice(__lowerCAmelCase		,   offset + self.batch_size						)		,   indices=self.dataset._indices		,   )
         _lowerCamelCase		:							Any            =					batch.to_pandas().to_json(
             path_or_buf=__lowerCAmelCase		,   orient=__lowerCAmelCase		,   lines=__lowerCAmelCase		,   index=__lowerCAmelCase		,   **__lowerCAmelCase						)
         if not json_str.endswith('''\n'''						):
                json_str += "\n"
         return json_str.encode(self.encoding						)




  def 		SCREAMING_SNAKE_CASE							(					self	:		List[str]		,   __lowerCAmelCase	:		BinaryIO		,   __lowerCAmelCase	:		List[str]		,   __lowerCAmelCase	:		List[Any]		,   __lowerCAmelCase	:		Optional[int]		,   **__lowerCAmelCase	:		Optional[Any]		,   ):






         """simple docstring"""






         _lowerCamelCase		:							int            =					0

         if self.num_proc is None or self.num_proc == 1:
                for offset in logging.tqdm(
                    range(0		,   len(self.dataset						)		,   self.batch_size						)		,   unit='''ba'''		,   disable=not logging.is_progress_bar_enabled()		,   desc='''Creating json from Arrow format'''		,   ):
                       _lowerCamelCase		:							Tuple            =					self._batch_json((offset, orient, lines, index, to_json_kwargs)						)
                       written += file_obj.write(__lowerCAmelCase						)
         else:
                _lowerCamelCase     ,       _lowerCamelCase		:							str            =					len(self.dataset						), self.batch_size
                with multiprocessing.Pool(self.num_proc						) as pool:
                       for json_str in logging.tqdm(
                           pool.imap(
                               self._batch_json		,   [(offset, orient, lines, index, to_json_kwargs) for offset in range(0		,   __lowerCAmelCase		,   __lowerCAmelCase						)]		,   )		,   total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size		,   unit='''ba'''		,   disable=not logging.is_progress_bar_enabled()		,   desc='''Creating json from Arrow format'''		,   ):
                              written += file_obj.write(__lowerCAmelCase						)

         return written









import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers.testing_utils import require_vision
from transformers.utils import is_vision_available


if is_vision_available():
       from PIL import Image

       from transformers import (
           AutoProcessor,
           BertTokenizerFast,
           BlipImageProcessor,
           GPTaTokenizer,
           InstructBlipProcessor,
           PreTrainedTokenizerFast,
       )







@require_vision
class       __A							(   unittest.TestCase	):




      """simple docstring"""







      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  List[Any]						=		tempfile.mkdtemp()

             _lowerCamelCase				:  List[str]						=		BlipImageProcessor()
             _lowerCamelCase				:  Union[str, Any]						=		GPTaTokenizer.from_pretrained('''hf-internal-testing/tiny-random-GPT2Model''')
             _lowerCamelCase				:  Any						=		BertTokenizerFast.from_pretrained('''hf-internal-testing/tiny-random-bert''')

             _lowerCamelCase				:  Any						=		InstructBlipProcessor(a__   ,				a__   ,				a__)

             processor.save_pretrained(self.tmpdirname)


      def 					__snake_case      (		self   ,				**a__):



             """simple docstring"""





             return AutoProcessor.from_pretrained(self.tmpdirname   ,				**a__).tokenizer


      def 					__snake_case      (		self   ,				**a__):



             """simple docstring"""





             return AutoProcessor.from_pretrained(self.tmpdirname   ,				**a__).image_processor


      def 					__snake_case      (		self   ,				**a__):



             """simple docstring"""





             return AutoProcessor.from_pretrained(self.tmpdirname   ,				**a__).qformer_tokenizer


      def 					__snake_case      (		self):



             """simple docstring"""





             shutil.rmtree(self.tmpdirname)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  List[Any]						=		[np.random.randint(255   ,				size=(3, 30, 400)   ,				dtype=np.uinta)]

             _lowerCamelCase				:  List[str]						=		[Image.fromarray(np.moveaxis(a__   ,				0   ,				-1)) for x in image_inputs]

             return image_inputs


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[Any]						=		InstructBlipProcessor(
                 tokenizer=self.get_tokenizer()   ,				image_processor=self.get_image_processor()   ,				qformer_tokenizer=self.get_qformer_tokenizer()   ,				)
             processor.save_pretrained(self.tmpdirname)

             _lowerCamelCase				:  Optional[int]						=		self.get_tokenizer(bos_token='''(BOS)'''   ,				eos_token='''(EOS)''')
             _lowerCamelCase				:  str						=		self.get_image_processor(do_normalize=a__   ,				padding_value=1.0)

             _lowerCamelCase				:  Tuple						=		InstructBlipProcessor.from_pretrained(
                 self.tmpdirname   ,				bos_token='''(BOS)'''   ,				eos_token='''(EOS)'''   ,				do_normalize=a__   ,				padding_value=1.0)

             self.assertEqual(processor.tokenizer.get_vocab()   ,				tokenizer_add_kwargs.get_vocab())
             self.assertIsInstance(processor.tokenizer   ,				a__)

             self.assertEqual(processor.image_processor.to_json_string()   ,				image_processor_add_kwargs.to_json_string())
             self.assertIsInstance(processor.image_processor   ,				a__)
             self.assertIsInstance(processor.qformer_tokenizer   ,				a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  List[str]						=		self.get_image_processor()
             _lowerCamelCase				:  Optional[int]						=		self.get_tokenizer()
             _lowerCamelCase				:  Any						=		self.get_qformer_tokenizer()

             _lowerCamelCase				:  Dict						=		InstructBlipProcessor(
                 tokenizer=a__   ,				image_processor=a__   ,				qformer_tokenizer=a__)

             _lowerCamelCase				:  List[Any]						=		self.prepare_image_inputs()

             _lowerCamelCase				:  List[str]						=		image_processor(a__   ,				return_tensors='''np''')
             _lowerCamelCase				:  List[Any]						=		processor(images=a__   ,				return_tensors='''np''')

             for key in input_feat_extract.keys():
                    self.assertAlmostEqual(input_feat_extract[key].sum()   ,				input_processor[key].sum()   ,				delta=1e-2)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Any						=		self.get_image_processor()
             _lowerCamelCase				:  Dict						=		self.get_tokenizer()
             _lowerCamelCase				:  int						=		self.get_qformer_tokenizer()

             _lowerCamelCase				:  Tuple						=		InstructBlipProcessor(
                 tokenizer=a__   ,				image_processor=a__   ,				qformer_tokenizer=a__)

             _lowerCamelCase				:  List[Any]						=		'''lower newer'''

             _lowerCamelCase				:  Any						=		processor(text=a__)

             _lowerCamelCase				:  Optional[int]						=		tokenizer(a__   ,				return_token_type_ids=a__)
             _lowerCamelCase				:  Optional[Any]						=		qformer_tokenizer(a__   ,				return_token_type_ids=a__)

             for key in encoded_tokens.keys():
                    self.assertListEqual(encoded_tokens[key]   ,				encoded_processor[key])

             for key in encoded_tokens_qformer.keys():
                    self.assertListEqual(encoded_tokens_qformer[key]   ,				encoded_processor['''qformer_''' + key])


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  List[Any]						=		self.get_image_processor()
             _lowerCamelCase				:  Optional[int]						=		self.get_tokenizer()
             _lowerCamelCase				:  Optional[Any]						=		self.get_qformer_tokenizer()

             _lowerCamelCase				:  List[Any]						=		InstructBlipProcessor(
                 tokenizer=a__   ,				image_processor=a__   ,				qformer_tokenizer=a__)

             _lowerCamelCase				:  List[Any]						=		'''lower newer'''
             _lowerCamelCase				:  Tuple						=		self.prepare_image_inputs()

             _lowerCamelCase				:  Tuple						=		processor(text=a__   ,				images=a__)

             self.assertListEqual(
                 list(inputs.keys())   ,				['''input_ids''', '''attention_mask''', '''qformer_input_ids''', '''qformer_attention_mask''', '''pixel_values''']   ,				)

             # test if it raises when no input is passed
             with pytest.raises(a__):
                    processor()


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  int						=		self.get_image_processor()
             _lowerCamelCase				:  List[Any]						=		self.get_tokenizer()
             _lowerCamelCase				:  List[Any]						=		self.get_qformer_tokenizer()

             _lowerCamelCase				:  Optional[Any]						=		InstructBlipProcessor(
                 tokenizer=a__   ,				image_processor=a__   ,				qformer_tokenizer=a__)

             _lowerCamelCase				:  List[Any]						=		[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

             _lowerCamelCase				:  Tuple						=		processor.batch_decode(a__)
             _lowerCamelCase				:  str						=		tokenizer.batch_decode(a__)

             self.assertListEqual(a__   ,				a__)



      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  List[str]						=		self.get_image_processor()
             _lowerCamelCase				:  Dict						=		self.get_tokenizer()
             _lowerCamelCase				:  str						=		self.get_qformer_tokenizer()

             _lowerCamelCase				:  str						=		InstructBlipProcessor(
                 tokenizer=a__   ,				image_processor=a__   ,				qformer_tokenizer=a__)

             _lowerCamelCase				:  str						=		'''lower newer'''
             _lowerCamelCase				:  str						=		self.prepare_image_inputs()

             _lowerCamelCase				:  List[Any]						=		processor(text=a__   ,				images=a__)

             self.assertListEqual(
                 list(inputs.keys())   ,				['''input_ids''', '''attention_mask''', '''qformer_input_ids''', '''qformer_attention_mask''', '''pixel_values''']   ,				)

