






import copy
import unittest

from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
          MODEL_FOR_QUESTION_ANSWERING_MAPPING,
          MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
          MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
          LayoutLMvaConfig,
          LayoutLMvaForQuestionAnswering,
          LayoutLMvaForSequenceClassification,
          LayoutLMvaForTokenClassification,
          LayoutLMvaModel,
      )
      from transformers.models.layoutlmva.modeling_layoutlmva import LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
      from PIL import Image

      from transformers import LayoutLMvaImageProcessor







class 				UpperCAmelCase_  :




     def __init__(				self	,						_lowerCAmelCase	,						_lowerCAmelCase=2	,						_lowerCAmelCase=3	,						_lowerCAmelCase=4	,						_lowerCAmelCase=2	,						_lowerCAmelCase=7	,						_lowerCAmelCase=True	,						_lowerCAmelCase=True	,						_lowerCAmelCase=True	,						_lowerCAmelCase=True	,						_lowerCAmelCase=99	,						_lowerCAmelCase=36	,						_lowerCAmelCase=3	,						_lowerCAmelCase=4	,						_lowerCAmelCase=37	,						_lowerCAmelCase="gelu"	,						_lowerCAmelCase=0.1	,						_lowerCAmelCase=0.1	,						_lowerCAmelCase=512	,						_lowerCAmelCase=16	,						_lowerCAmelCase=2	,						_lowerCAmelCase=0.0_2	,						_lowerCAmelCase=6	,						_lowerCAmelCase=6	,						_lowerCAmelCase=3	,						_lowerCAmelCase=4	,						_lowerCAmelCase=None	,						_lowerCAmelCase=1000	,						):
          UpperCAmelCase__      :  List[str]				      =			parent
          UpperCAmelCase__      :  Dict				      =			batch_size
          UpperCAmelCase__      :  Any				      =			num_channels
          UpperCAmelCase__      :  str				      =			image_size
          UpperCAmelCase__      :  List[str]				      =			patch_size
          UpperCAmelCase__      :  Union[str, Any]				      =			text_seq_length
          UpperCAmelCase__      :  Optional[Any]				      =			is_training
          UpperCAmelCase__      :  int				      =			use_input_mask
          UpperCAmelCase__      :  Tuple				      =			use_token_type_ids
          UpperCAmelCase__      :  List[Any]				      =			use_labels
          UpperCAmelCase__      :  Tuple				      =			vocab_size
          UpperCAmelCase__      :  List[str]				      =			hidden_size
          UpperCAmelCase__      :  List[Any]				      =			num_hidden_layers
          UpperCAmelCase__      :  str				      =			num_attention_heads
          UpperCAmelCase__      :  Dict				      =			intermediate_size
          UpperCAmelCase__      :  Union[str, Any]				      =			hidden_act
          UpperCAmelCase__      :  List[Any]				      =			hidden_dropout_prob
          UpperCAmelCase__      :  Tuple				      =			attention_probs_dropout_prob
          UpperCAmelCase__      :  Any				      =			max_position_embeddings
          UpperCAmelCase__      :  Union[str, Any]				      =			type_vocab_size
          UpperCAmelCase__      :  List[str]				      =			type_sequence_label_size
          UpperCAmelCase__      :  Union[str, Any]				      =			initializer_range
          UpperCAmelCase__      :  str				      =			coordinate_size
          UpperCAmelCase__      :  Union[str, Any]				      =			shape_size
          UpperCAmelCase__      :  Optional[Any]				      =			num_labels
          UpperCAmelCase__      :  Optional[int]				      =			num_choices
          UpperCAmelCase__      :  Tuple				      =			scope
          UpperCAmelCase__      :  Optional[Any]				      =			range_bbox

          # LayoutLMv3's sequence length equals the number of text tokens + number of patches + 1 (we add 1 for the CLS token)
          UpperCAmelCase__      :  List[Any]				      =			text_seq_length
          UpperCAmelCase__      :  int				      =			(image_size // patch_size) ** 2 + 1
          UpperCAmelCase__      :  str				      =			self.text_seq_length + self.image_seq_length




     def   __UpperCAmelCase  (				self		):
          UpperCAmelCase__      :  List[str]				      =			ids_tensor([self.batch_size, self.text_seq_length]	,						self.vocab_size		)

          UpperCAmelCase__      :  str				      =			ids_tensor([self.batch_size, self.text_seq_length, 4]	,						self.range_bbox		)
          # Ensure that bbox is legal
          for i in range(bbox.shape[0]		):
               for j in range(bbox.shape[1]		):
                    if bbox[i, j, 3] < bbox[i, j, 1]:
                         UpperCAmelCase__      :  Tuple				      =			bbox[i, j, 3]
                         UpperCAmelCase__      :  int				      =			bbox[i, j, 1]
                         UpperCAmelCase__      :  Any				      =			t
                    if bbox[i, j, 2] < bbox[i, j, 0]:
                         UpperCAmelCase__      :  int				      =			bbox[i, j, 2]
                         UpperCAmelCase__      :  Optional[int]				      =			bbox[i, j, 0]
                         UpperCAmelCase__      :  Any				      =			t

          UpperCAmelCase__      :  Tuple				      =			floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]		)

          UpperCAmelCase__      :  Optional[int]				      =			None
          if self.use_input_mask:
               UpperCAmelCase__      :  Optional[int]				      =			random_attention_mask([self.batch_size, self.text_seq_length]		)

          UpperCAmelCase__      :  Dict				      =			None
          if self.use_token_type_ids:
               UpperCAmelCase__      :  Optional[int]				      =			ids_tensor([self.batch_size, self.text_seq_length]	,						self.type_vocab_size		)

          UpperCAmelCase__      :  int				      =			None
          UpperCAmelCase__      :  Union[str, Any]				      =			None
          if self.use_labels:
               UpperCAmelCase__      :  List[str]				      =			ids_tensor([self.batch_size]	,						self.type_sequence_label_size		)
               UpperCAmelCase__      :  Any				      =			ids_tensor([self.batch_size, self.text_seq_length]	,						self.num_labels		)

          UpperCAmelCase__      :  List[Any]				      =			LayoutLMvaConfig(
              vocab_size=self.vocab_size	,						hidden_size=self.hidden_size	,						num_hidden_layers=self.num_hidden_layers	,						num_attention_heads=self.num_attention_heads	,						intermediate_size=self.intermediate_size	,						hidden_act=self.hidden_act	,						hidden_dropout_prob=self.hidden_dropout_prob	,						attention_probs_dropout_prob=self.attention_probs_dropout_prob	,						max_position_embeddings=self.max_position_embeddings	,						type_vocab_size=self.type_vocab_size	,						initializer_range=self.initializer_range	,						coordinate_size=self.coordinate_size	,						shape_size=self.shape_size	,						input_size=self.image_size	,						patch_size=self.patch_size	,						)

          return config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase		):
          UpperCAmelCase__      :  Any				      =			LayoutLMvaModel(config=UpperCamelCase__		)
          model.to(UpperCamelCase__		)
          model.eval()

          # text + image
          UpperCAmelCase__      :  List[str]				      =			model(UpperCamelCase__	,						pixel_values=UpperCamelCase__		)
          UpperCAmelCase__      :  Tuple				      =			model(
              UpperCamelCase__	,						bbox=UpperCamelCase__	,						pixel_values=UpperCamelCase__	,						attention_mask=UpperCamelCase__	,						token_type_ids=UpperCamelCase__		)
          UpperCAmelCase__      :  Optional[int]				      =			model(UpperCamelCase__	,						bbox=UpperCamelCase__	,						pixel_values=UpperCamelCase__	,						token_type_ids=UpperCamelCase__		)
          UpperCAmelCase__      :  List[Any]				      =			model(UpperCamelCase__	,						bbox=UpperCamelCase__	,						pixel_values=UpperCamelCase__		)

          self.parent.assertEqual(result.last_hidden_state.shape	,						(self.batch_size, self.seq_length, self.hidden_size)		)

          # text only
          UpperCAmelCase__      :  List[str]				      =			model(UpperCamelCase__		)
          self.parent.assertEqual(
              result.last_hidden_state.shape	,						(self.batch_size, self.text_seq_length, self.hidden_size)		)

          # image only
          UpperCAmelCase__      :  Union[str, Any]				      =			model(pixel_values=UpperCamelCase__		)
          self.parent.assertEqual(
              result.last_hidden_state.shape	,						(self.batch_size, self.image_seq_length, self.hidden_size)		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase		):
          UpperCAmelCase__      :  Tuple				      =			self.num_labels
          UpperCAmelCase__      :  int				      =			LayoutLMvaForSequenceClassification(UpperCamelCase__		)
          model.to(UpperCamelCase__		)
          model.eval()
          UpperCAmelCase__      :  Optional[Any]				      =			model(
              UpperCamelCase__	,						bbox=UpperCamelCase__	,						pixel_values=UpperCamelCase__	,						attention_mask=UpperCamelCase__	,						token_type_ids=UpperCamelCase__	,						labels=UpperCamelCase__	,						)
          self.parent.assertEqual(result.logits.shape	,						(self.batch_size, self.num_labels)		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase		):
          UpperCAmelCase__      :  Any				      =			self.num_labels
          UpperCAmelCase__      :  Union[str, Any]				      =			LayoutLMvaForTokenClassification(config=UpperCamelCase__		)
          model.to(UpperCamelCase__		)
          model.eval()
          UpperCAmelCase__      :  Optional[Any]				      =			model(
              UpperCamelCase__	,						bbox=UpperCamelCase__	,						pixel_values=UpperCamelCase__	,						attention_mask=UpperCamelCase__	,						token_type_ids=UpperCamelCase__	,						labels=UpperCamelCase__	,						)
          self.parent.assertEqual(result.logits.shape	,						(self.batch_size, self.text_seq_length, self.num_labels)		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase		):
          UpperCAmelCase__      :  Tuple				      =			LayoutLMvaForQuestionAnswering(config=UpperCamelCase__		)
          model.to(UpperCamelCase__		)
          model.eval()
          UpperCAmelCase__      :  List[str]				      =			model(
              UpperCamelCase__	,						bbox=UpperCamelCase__	,						pixel_values=UpperCamelCase__	,						attention_mask=UpperCamelCase__	,						token_type_ids=UpperCamelCase__	,						start_positions=UpperCamelCase__	,						end_positions=UpperCamelCase__	,						)
          self.parent.assertEqual(result.start_logits.shape	,						(self.batch_size, self.seq_length)		)
          self.parent.assertEqual(result.end_logits.shape	,						(self.batch_size, self.seq_length)		)




     def   __UpperCAmelCase  (				self		):
          UpperCAmelCase__      :  str				      =			self.prepare_config_and_inputs()
          (
              (
              UpperCAmelCase__	
          )      ,		(
              UpperCAmelCase__	
          )      ,		(
              UpperCAmelCase__	
          )      ,		(
              UpperCAmelCase__	
          )      ,		(
              UpperCAmelCase__	
          )      ,		(
              UpperCAmelCase__	
          )      ,		(
              UpperCAmelCase__	
          )      ,		(
              UpperCAmelCase__	
          )      ,			
          )      :  Any				      =			config_and_inputs
          UpperCAmelCase__      :  Optional[Any]				      =			{
              """input_ids""": input_ids,
              """bbox""": bbox,
              """pixel_values""": pixel_values,
              """token_type_ids""": token_type_ids,
              """attention_mask""": input_mask,
          }
          return config, inputs_dict







@require_torch
class 				UpperCAmelCase_  (		__A							,      __A							,      unittest.TestCase		):
     __lowerCamelCase      				=  False
     __lowerCamelCase      				=  False
     __lowerCamelCase      				=  False

     __lowerCamelCase      				=  (
         (
             LayoutLMvaModel,
             LayoutLMvaForSequenceClassification,
             LayoutLMvaForTokenClassification,
             LayoutLMvaForQuestionAnswering,
         )
         if is_torch_available()
         else ()
     )
     __lowerCamelCase      				=  (
         {'''document-question-answering''': LayoutLMvaForQuestionAnswering, '''feature-extraction''': LayoutLMvaModel}
         if is_torch_available()
         else {}
     )




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase		):
          return True




     def   __UpperCAmelCase  (				self		):
          UpperCAmelCase__      :  List[str]				      =			LayoutLMvaModelTester(self		)
          UpperCAmelCase__      :  Optional[int]				      =			ConfigTester(self	,						config_class=UpperCamelCase__	,						hidden_size=37		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase=False		):
          UpperCAmelCase__      :  Any				      =			copy.deepcopy(UpperCamelCase__		)
          if model_class in get_values(UpperCamelCase__		):
               UpperCAmelCase__      :  List[str]				      =			{
                   k: v.unsqueeze(1		).expand(-1	,						self.model_tester.num_choices	,						-1		).contiguous()
                   if isinstance(UpperCamelCase__	,						torch.Tensor		) and v.ndim > 1
                   else v
                   for k, v in inputs_dict.items()
               }
          if return_labels:
               if model_class in get_values(UpperCamelCase__		):
                    UpperCAmelCase__      :  Any				      =			torch.ones(self.model_tester.batch_size	,						dtype=torch.long	,						device=UpperCamelCase__		)
               elif model_class in get_values(UpperCamelCase__		):
                    UpperCAmelCase__      :  Dict				      =			torch.zeros(
                        self.model_tester.batch_size	,						dtype=torch.long	,						device=UpperCamelCase__		)
                    UpperCAmelCase__      :  Dict				      =			torch.zeros(
                        self.model_tester.batch_size	,						dtype=torch.long	,						device=UpperCamelCase__		)
               elif model_class in [
                   *get_values(UpperCamelCase__		),
               ]:
                    UpperCAmelCase__      :  List[str]				      =			torch.zeros(
                        self.model_tester.batch_size	,						dtype=torch.long	,						device=UpperCamelCase__		)
               elif model_class in [
                   *get_values(UpperCamelCase__		),
               ]:
                    UpperCAmelCase__      :  Optional[Any]				      =			torch.zeros(
                        (self.model_tester.batch_size, self.model_tester.text_seq_length)	,						dtype=torch.long	,						device=UpperCamelCase__	,						)

          return inputs_dict




     def   __UpperCAmelCase  (				self		):
          self.config_tester.run_common_tests()




     def   __UpperCAmelCase  (				self		):
          UpperCAmelCase__      :  Union[str, Any]				      =			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*UpperCamelCase__		)




     def   __UpperCAmelCase  (				self		):
          UpperCAmelCase__      :  List[Any]				      =			self.model_tester.prepare_config_and_inputs()
          for type in ["absolute", "relative_key", "relative_key_query"]:
               UpperCAmelCase__      :  Optional[Any]				      =			type
               self.model_tester.create_and_check_model(*UpperCamelCase__		)




     def   __UpperCAmelCase  (				self		):
          UpperCAmelCase__      :  Dict				      =			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*UpperCamelCase__		)




     def   __UpperCAmelCase  (				self		):
          UpperCAmelCase__      :  Union[str, Any]				      =			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*UpperCamelCase__		)




     def   __UpperCAmelCase  (				self		):
          UpperCAmelCase__      :  Dict				      =			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*UpperCamelCase__		)




     @slow
     def   __UpperCAmelCase  (				self		):


          for model_name in LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               UpperCAmelCase__      :  List[Any]				      =			LayoutLMvaModel.from_pretrained(UpperCamelCase__		)
               self.assertIsNotNone(UpperCamelCase__		)





def  _lowerCamelCase						(						) ->			int:

     '''simple docstring'''
     UpperCAmelCase__      :  Any				      =			Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""       )
     return image







@require_torch
class 				UpperCAmelCase_  (		unittest.TestCase		):




     @cached_property
     def   __UpperCAmelCase  (				self		):
          return LayoutLMvaImageProcessor(apply_ocr=UpperCamelCase__		) if is_vision_available() else None




     @slow
     def   __UpperCAmelCase  (				self		):
          UpperCAmelCase__      :  Optional[int]				      =			LayoutLMvaModel.from_pretrained("""microsoft/layoutlmv3-base"""		).to(UpperCamelCase__		)

          UpperCAmelCase__      :  Union[str, Any]				      =			self.default_image_processor
          UpperCAmelCase__      :  Tuple				      =			prepare_img()
          UpperCAmelCase__      :  Union[str, Any]				      =			image_processor(images=UpperCamelCase__	,						return_tensors="""pt"""		).pixel_values.to(UpperCamelCase__		)

          UpperCAmelCase__      :  Optional[int]				      =			torch.tensor([[1, 2]]		)
          UpperCAmelCase__      :  List[Any]				      =			torch.tensor([[1, 2, 3, 4], [5, 6, 7, 8]]		).unsqueeze(0		)

          # forward pass
          UpperCAmelCase__      :  Tuple				      =			model(
              input_ids=input_ids.to(UpperCamelCase__		)	,						bbox=bbox.to(UpperCamelCase__		)	,						pixel_values=pixel_values.to(UpperCamelCase__		)	,						)

          # verify the logits
          UpperCAmelCase__      :  Dict				      =			torch.Size((1, 199, 768)		)
          self.assertEqual(outputs.last_hidden_state.shape	,						UpperCamelCase__		)

          UpperCAmelCase__      :  Any				      =			torch.tensor(
              [[-0.0_5_2_9, 0.3_6_1_8, 0.1_6_3_2], [-0.1_5_8_7, -0.1_6_6_7, -0.0_4_0_0], [-0.1_5_5_7, -0.1_6_7_1, -0.0_5_0_5]]		).to(UpperCamelCase__		)

          self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]	,						UpperCamelCase__	,						atol=1e-4		)		)














'''simple docstring'''







import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader




class   __SCREAMING_SNAKE_CASE       (__A       ):


  """simple docstring"""


  def __init__(			self   ,	UpperCamelCase__   ,	UpperCamelCase__ = None   ,	UpperCamelCase__ = None   ,	UpperCamelCase__ = None   ,	UpperCamelCase__ = False   ,	UpperCamelCase__ = False   ,	UpperCamelCase__ = None   ,	UpperCamelCase__ = None   ,	**UpperCamelCase__   ,	):






         """simple docstring"""


         super().__init__(
             UpperCamelCase__   ,	split=UpperCamelCase__   ,	features=UpperCamelCase__   ,	cache_dir=UpperCamelCase__   ,	keep_in_memory=UpperCamelCase__   ,	streaming=UpperCamelCase__   ,	num_proc=UpperCamelCase__   ,	**UpperCamelCase__   ,	)
         a_						     =			field
         a_						     =			path_or_paths if isinstance(UpperCamelCase__   ,	UpperCamelCase__	) else {self.split: path_or_paths}
         a_						     =			Json(
             cache_dir=UpperCamelCase__   ,	data_files=UpperCamelCase__   ,	features=UpperCamelCase__   ,	field=UpperCamelCase__   ,	**UpperCamelCase__   ,	)







  def    _a       (			self	):






         """simple docstring"""


         if self.streaming:
                a_						     =			self.builder.as_streaming_dataset(split=self.split	)
         # Build regular (map-style) dataset
         else:
                a_						     =			None
                a_						     =			None
                a_						     =			None
                a_						     =			None

                self.builder.download_and_prepare(
                    download_config=UpperCamelCase__   ,	download_mode=UpperCamelCase__   ,	verification_mode=UpperCamelCase__   ,	base_path=UpperCamelCase__   ,	num_proc=self.num_proc   ,	)
                a_						     =			self.builder.as_dataset(
                    split=self.split   ,	verification_mode=UpperCamelCase__   ,	in_memory=self.keep_in_memory	)
         return dataset





class   __SCREAMING_SNAKE_CASE       :


  """simple docstring"""


  def __init__(			self   ,	UpperCamelCase__   ,	UpperCamelCase__   ,	UpperCamelCase__ = None   ,	UpperCamelCase__ = None   ,	**UpperCamelCase__   ,	):






         """simple docstring"""


         if num_proc is not None and num_proc <= 0:
                raise ValueError(f'num_proc {num_proc} must be an integer > 0.'	)

         a_						     =			dataset
         a_						     =			path_or_buf
         a_						     =			batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
         a_						     =			num_proc
         a_						     =			'utf-8'
         a_						     =			to_json_kwargs







  def    _a       (			self	):






         """simple docstring"""


         a_						     =			self.to_json_kwargs.pop('path_or_buf'   ,	UpperCamelCase__	)
         a_						     =			self.to_json_kwargs.pop('orient'   ,	'records'	)
         a_						     =			self.to_json_kwargs.pop('lines'   ,	True if orient == 'records' else False	)
         a_						     =			self.to_json_kwargs.pop('index'   ,	False if orient in ['split', 'table'] else True	)
         a_						     =			self.to_json_kwargs.pop('compression'   ,	UpperCamelCase__	)

         if compression not in [None, "infer", "gzip", "bz2", "xz"]:
                raise NotImplementedError(f'`datasets` currently does not support {compression} compression'	)

         if isinstance(self.path_or_buf   ,	(str, bytes, os.PathLike)	):
                with fsspec.open(self.path_or_buf   ,	'wb'   ,	compression=UpperCamelCase__	) as buffer:
                       a_						     =			self._write(file_obj=UpperCamelCase__   ,	orient=UpperCamelCase__   ,	lines=UpperCamelCase__   ,	index=UpperCamelCase__   ,	**self.to_json_kwargs	)
         else:
                if compression:
                       raise NotImplementedError(
                           f'The compression parameter is not supported when writing to a buffer, but compression={compression}'
                           ' was passed. Please provide a local path instead.'	)
                a_						     =			self._write(
                    file_obj=self.path_or_buf   ,	orient=UpperCamelCase__   ,	lines=UpperCamelCase__   ,	index=UpperCamelCase__   ,	**self.to_json_kwargs	)
         return written







  def    _a       (			self   ,	UpperCamelCase__	):






         """simple docstring"""


         a_     ,	a_     ,	a_     ,	a_     ,	a_						     =			args

         a_						     =			query_table(
             table=self.dataset.data   ,	key=slice(UpperCamelCase__   ,	offset + self.batch_size	)   ,	indices=self.dataset._indices   ,	)
         a_						     =			batch.to_pandas().to_json(
             path_or_buf=UpperCamelCase__   ,	orient=UpperCamelCase__   ,	lines=UpperCamelCase__   ,	index=UpperCamelCase__   ,	**UpperCamelCase__	)
         if not json_str.endswith('\n'	):
                json_str += "\n"
         return json_str.encode(self.encoding	)







  def    _a       (			self   ,	UpperCamelCase__   ,	UpperCamelCase__   ,	UpperCamelCase__   ,	UpperCamelCase__   ,	**UpperCamelCase__   ,	):






         """simple docstring"""


         a_						     =			0

         if self.num_proc is None or self.num_proc == 1:
                for offset in logging.tqdm(
                    range(0   ,	len(self.dataset	)   ,	self.batch_size	)   ,	unit='ba'   ,	disable=not logging.is_progress_bar_enabled()   ,	desc='Creating json from Arrow format'   ,	):
                       a_						     =			self._batch_json((offset, orient, lines, index, to_json_kwargs)	)
                       written += file_obj.write(UpperCamelCase__	)
         else:
                a_     ,	a_						     =			len(self.dataset	), self.batch_size
                with multiprocessing.Pool(self.num_proc	) as pool:
                       for json_str in logging.tqdm(
                           pool.imap(
                               self._batch_json   ,	[(offset, orient, lines, index, to_json_kwargs) for offset in range(0   ,	UpperCamelCase__   ,	UpperCamelCase__	)]   ,	)   ,	total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size   ,	unit='ba'   ,	disable=not logging.is_progress_bar_enabled()   ,	desc='Creating json from Arrow format'   ,	):
                              written += file_obj.write(UpperCamelCase__	)

         return written




