
"""simple docstring"""






import inspect
import unittest

from transformers import MobileViTVaConfig
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import MobileViTVaForImageClassification, MobileViTVaForSemanticSegmentation, MobileViTVaModel
 from transformers.models.mobilevitva.modeling_mobilevitva import (
     MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST,
     make_divisible,
 )


if is_vision_available():
 from PIL import Image

 from transformers import MobileViTImageProcessor





class 				a__					(						A__       ):



     def  lowerCamelCase_						(							self :Tuple   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=self.config_class(**self.inputs_dict   )
      self.parent.assertTrue(hasattr(_lowerCamelCase      ,  'width_multiplier'   )   )





class 				a__					:



     def __init__(							self :Tuple      ,  _lowerCamelCase :List[Any]      ,  _lowerCamelCase :str=13      ,  _lowerCamelCase :Optional[Any]=64      ,  _lowerCamelCase :Optional[int]=2      ,  _lowerCamelCase :Union[str, Any]=3      ,  _lowerCamelCase :Optional[int]="swish"      ,  _lowerCamelCase :List[Any]=3      ,  _lowerCamelCase :Union[str, Any]=32      ,  _lowerCamelCase :Optional[Any]=0.1      ,  _lowerCamelCase :int=0.02      ,  _lowerCamelCase :List[str]=True      ,  _lowerCamelCase :List[Any]=True      ,  _lowerCamelCase :Union[str, Any]=10      ,  _lowerCamelCase :Dict=None      ,  _lowerCamelCase :Union[str, Any]=0.25      ,  _lowerCamelCase :Dict=0.0      ,  _lowerCamelCase :int=0.0      ,  ):







      '''simple docstring'''



      UpperCamelCase_     :					Tuple   	=parent
      UpperCamelCase_     :					Tuple   	=batch_size
      UpperCamelCase_     :					Tuple   	=image_size
      UpperCamelCase_     :					Any   	=patch_size
      UpperCamelCase_     :					Tuple   	=num_channels
      UpperCamelCase_     :					Optional[int]   	=make_divisible(512 * width_multiplier      ,  divisor=8   )
      UpperCamelCase_     :					Union[str, Any]   	=hidden_act
      UpperCamelCase_     :					Union[str, Any]   	=conv_kernel_size
      UpperCamelCase_     :					Any   	=output_stride
      UpperCamelCase_     :					Optional[Any]   	=classifier_dropout_prob
      UpperCamelCase_     :					List[Any]   	=use_labels
      UpperCamelCase_     :					int   	=is_training
      UpperCamelCase_     :					int   	=num_labels
      UpperCamelCase_     :					Any   	=initializer_range
      UpperCamelCase_     :					Any   	=scope
      UpperCamelCase_     :					Optional[Any]   	=width_multiplier
      UpperCamelCase_     :					Optional[int]   	=ffn_dropout
      UpperCamelCase_     :					int   	=attn_dropout



     def  lowerCamelCase_						(							self :Any   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]   )

      UpperCamelCase_     :					Tuple   	=None
      UpperCamelCase_     :					Union[str, Any]   	=None
      if self.use_labels:
       UpperCamelCase_     :					int   	=ids_tensor([self.batch_size]      ,  self.num_labels   )
       UpperCamelCase_     :					Any   	=ids_tensor([self.batch_size, self.image_size, self.image_size]      ,  self.num_labels   )

      UpperCamelCase_     :					str   	=self.get_config()

      return config, pixel_values, labels, pixel_labels



     def  lowerCamelCase_						(							self :List[str]   ):







      '''simple docstring'''



      return MobileViTVaConfig(
          image_size=self.image_size      ,  patch_size=self.patch_size      ,  num_channels=self.num_channels      ,  hidden_act=self.hidden_act      ,  conv_kernel_size=self.conv_kernel_size      ,  output_stride=self.output_stride      ,  classifier_dropout_prob=self.classifier_dropout_prob      ,  initializer_range=self.initializer_range      ,  width_multiplier=self.width_multiplier      ,  ffn_dropout=self.ffn_dropout_prob      ,  attn_dropout=self.attn_dropout_prob      ,  )



     def  lowerCamelCase_						(							self :Any      ,  _lowerCamelCase :str      ,  _lowerCamelCase :List[Any]      ,  _lowerCamelCase :Tuple      ,  _lowerCamelCase :List[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					Union[str, Any]   	=MobileViTVaModel(config=_lowerCamelCase   )
      model.to(_lowerCamelCase   )
      model.eval()
      UpperCamelCase_     :					int   	=model(_lowerCamelCase   )
      self.parent.assertEqual(
          result.last_hidden_state.shape      ,  (
              self.batch_size,
              self.last_hidden_size,
              self.image_size // self.output_stride,
              self.image_size // self.output_stride,
          )      ,  )



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :List[str]      ,  _lowerCamelCase :Any      ,  _lowerCamelCase :List[Any]      ,  _lowerCamelCase :int   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=self.num_labels
      UpperCamelCase_     :					Union[str, Any]   	=MobileViTVaForImageClassification(_lowerCamelCase   )
      model.to(_lowerCamelCase   )
      model.eval()
      UpperCamelCase_     :					List[Any]   	=model(_lowerCamelCase      ,  labels=_lowerCamelCase   )
      self.parent.assertEqual(result.logits.shape      ,  (self.batch_size, self.num_labels)   )



     def  lowerCamelCase_						(							self :Dict      ,  _lowerCamelCase :Dict      ,  _lowerCamelCase :List[str]      ,  _lowerCamelCase :Dict      ,  _lowerCamelCase :List[str]   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[str]   	=self.num_labels
      UpperCamelCase_     :					List[str]   	=MobileViTVaForSemanticSegmentation(_lowerCamelCase   )
      model.to(_lowerCamelCase   )
      model.eval()
      UpperCamelCase_     :					Dict   	=model(_lowerCamelCase   )
      self.parent.assertEqual(
          result.logits.shape      ,  (
              self.batch_size,
              self.num_labels,
              self.image_size // self.output_stride,
              self.image_size // self.output_stride,
          )      ,  )
      UpperCamelCase_     :					Union[str, Any]   	=model(_lowerCamelCase      ,  labels=_lowerCamelCase   )
      self.parent.assertEqual(
          result.logits.shape      ,  (
              self.batch_size,
              self.num_labels,
              self.image_size // self.output_stride,
              self.image_size // self.output_stride,
          )      ,  )




     def  lowerCamelCase_						(							self :Optional[int]   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[int]   	=self.prepare_config_and_inputs()
      UpperCamelCase_	,     UpperCamelCase_	,     UpperCamelCase_	,     UpperCamelCase_     :					List[Any]   	=config_and_inputs
      UpperCamelCase_     :					Any   	={'pixel_values': pixel_values}
      return config, inputs_dict





@require_torch
class 				a__					(						A__   ,							A__   ,							unittest.TestCase       ):
     UpperCAmelCase__         =      (
         (MobileViTVaModel, MobileViTVaForImageClassification, MobileViTVaForSemanticSegmentation)
         if is_torch_available()
         else ()
     )

     UpperCAmelCase__         =      (
         {
             '''feature-extraction''': MobileViTVaModel,
             '''image-classification''': MobileViTVaForImageClassification,
             '''image-segmentation''': MobileViTVaForSemanticSegmentation,
         }
         if is_torch_available()
         else {}
     )

     UpperCAmelCase__         =      False
     UpperCAmelCase__         =      False
     UpperCAmelCase__         =      False
     UpperCAmelCase__         =      False



     def  lowerCamelCase_						(							self :Dict   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=MobileViTVaModelTester(self   )
      UpperCamelCase_     :					Any   	=MobileViTVaConfigTester(self      ,  config_class=_lowerCamelCase      ,  has_text_modality=_lowerCamelCase   )



     def  lowerCamelCase_						(							self :str   ):







      '''simple docstring'''



      self.config_tester.run_common_tests()



     @unittest.skip(reason='MobileViTV2 does not use inputs_embeds'   )
     def  lowerCamelCase_						(							self :List[str]   ):







      '''simple docstring'''



      pass



     @unittest.skip(reason='MobileViTV2 does not support input and output embeddings'   )
     def  lowerCamelCase_						(							self :Union[str, Any]   ):







      '''simple docstring'''



      pass



     @unittest.skip(reason='MobileViTV2 does not output attentions'   )
     def  lowerCamelCase_						(							self :int   ):







      '''simple docstring'''



      pass



     @require_torch_multi_gpu
     @unittest.skip(reason='Got `CUDA error: misaligned address` for tests after this one being run.'   )
     def  lowerCamelCase_						(							self :List[str]   ):







      '''simple docstring'''



      pass



     @unittest.skip('Will be fixed soon by reducing the size of the model used for common tests.'   )
     def  lowerCamelCase_						(							self :Dict   ):







      '''simple docstring'''



      pass



     def  lowerCamelCase_						(							self :int   ):







      '''simple docstring'''



      UpperCamelCase_	,     UpperCamelCase_     :					Optional[Any]   	=self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       UpperCamelCase_     :					Optional[int]   	=model_class(_lowerCamelCase   )
       UpperCamelCase_     :					int   	=inspect.signature(model.forward   )
       # signature.parameters is an OrderedDict => so arg_names order is deterministic
       UpperCamelCase_     :					Tuple   	=[*signature.parameters.keys()]

       UpperCamelCase_     :					Dict   	=['pixel_values']
       self.assertListEqual(arg_names[:1]      ,  _lowerCamelCase   )



     def  lowerCamelCase_						(							self :Union[str, Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*_lowerCamelCase   )



     def  lowerCamelCase_						(							self :Optional[Any]   ):







      '''simple docstring'''



      def check_hidden_states_output(_lowerCamelCase :Union[str, Any]      ,  _lowerCamelCase :Tuple      ,  _lowerCamelCase :Any   ):
       UpperCamelCase_     :					Optional[int]   	=model_class(_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()

       with torch.no_grad():
        UpperCamelCase_     :					Optional[Any]   	=model(**self._prepare_for_class(_lowerCamelCase      ,  _lowerCamelCase   )   )

       UpperCamelCase_     :					str   	=outputs.hidden_states

       UpperCamelCase_     :					int   	=5
       self.assertEqual(len(_lowerCamelCase   )      ,  _lowerCamelCase   )

       # MobileViTV2's feature maps are of shape (batch_size, num_channels, height, width)
       # with the width and height being successively divided by 2.
       UpperCamelCase_     :					Dict   	=2
       for i in range(len(_lowerCamelCase   )   ):
        self.assertListEqual(
            list(hidden_states[i].shape[-2:]   )      ,  [self.model_tester.image_size // divisor, self.model_tester.image_size // divisor]      ,  )
        divisor *= 2

       self.assertEqual(self.model_tester.output_stride      ,  divisor // 2   )

      UpperCamelCase_	,     UpperCamelCase_     :					int   	=self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       UpperCamelCase_     :					Tuple   	=True
       check_hidden_states_output(_lowerCamelCase      ,  _lowerCamelCase      ,  _lowerCamelCase   )

       # check that output_hidden_states also work using config
       del inputs_dict["output_hidden_states"]
       UpperCamelCase_     :					Any   	=True

       check_hidden_states_output(_lowerCamelCase      ,  _lowerCamelCase      ,  _lowerCamelCase   )



     def  lowerCamelCase_						(							self :str   ):







      '''simple docstring'''



      UpperCamelCase_     :					Tuple   	=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_image_classification(*_lowerCamelCase   )



     def  lowerCamelCase_						(							self :Union[str, Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_semantic_segmentation(*_lowerCamelCase   )




     @slow
     def  lowerCamelCase_						(							self :Union[str, Any]   ):







      '''simple docstring'''



      for model_name in MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       UpperCamelCase_     :					Dict   	=MobileViTVaModel.from_pretrained(_lowerCamelCase   )
       self.assertIsNotNone(_lowerCamelCase   )




def       A_  (				):
 UpperCamelCase_     :					Any   	=Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'	)
 return image





@require_torch
@require_vision
class 				a__					(						unittest.TestCase       ):



     @cached_property
     def  lowerCamelCase_						(							self :Tuple   ):







      '''simple docstring'''



      return (
          MobileViTImageProcessor.from_pretrained('apple/mobilevitv2-1.0-imagenet1k-256'   )
          if is_vision_available()
          else None
      )



     @slow
     def  lowerCamelCase_						(							self :List[str]   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=MobileViTVaForImageClassification.from_pretrained('apple/mobilevitv2-1.0-imagenet1k-256'   ).to(
          _lowerCamelCase   )

      UpperCamelCase_     :					Optional[Any]   	=self.default_image_processor
      UpperCamelCase_     :					int   	=prepare_img()
      UpperCamelCase_     :					str   	=image_processor(images=_lowerCamelCase      ,  return_tensors='pt'   ).to(_lowerCamelCase   )

      # forward pass
      with torch.no_grad():
       UpperCamelCase_     :					Optional[int]   	=model(**_lowerCamelCase   )

      # verify the logits
      UpperCamelCase_     :					Tuple   	=torch.Size((1, 1_000)   )
      self.assertEqual(outputs.logits.shape      ,  _lowerCamelCase   )

      UpperCamelCase_     :					Optional[Any]   	=torch.tensor([-1.6_3_3_6E0_0, -7.3_2_0_4E-0_2, -5.1_8_8_3E-0_1]   ).to(_lowerCamelCase   )

      self.assertTrue(torch.allclose(outputs.logits[0, :3]      ,  _lowerCamelCase      ,  atol=1E-4   )   )



     @slow
     def  lowerCamelCase_						(							self :Dict   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[str]   	=MobileViTVaForSemanticSegmentation.from_pretrained('shehan97/mobilevitv2-1.0-voc-deeplabv3'   )
      UpperCamelCase_     :					Optional[Any]   	=model.to(_lowerCamelCase   )

      UpperCamelCase_     :					Any   	=MobileViTImageProcessor.from_pretrained('shehan97/mobilevitv2-1.0-voc-deeplabv3'   )

      UpperCamelCase_     :					List[Any]   	=prepare_img()
      UpperCamelCase_     :					Optional[int]   	=image_processor(images=_lowerCamelCase      ,  return_tensors='pt'   ).to(_lowerCamelCase   )

      # forward pass
      with torch.no_grad():
       UpperCamelCase_     :					Optional[int]   	=model(**_lowerCamelCase   )
      UpperCamelCase_     :					Union[str, Any]   	=outputs.logits

      # verify the logits
      UpperCamelCase_     :					Dict   	=torch.Size((1, 21, 32, 32)   )
      self.assertEqual(logits.shape      ,  _lowerCamelCase   )

      UpperCamelCase_     :					int   	=torch.tensor(
          [
              [[7.0863, 7.1525, 6.8201], [6.6931, 6.8770, 6.8933], [6.2978, 7.0366, 6.9636]],
              [[-3.7134, -3.6712, -3.6675], [-3.5825, -3.3549, -3.4777], [-3.3435, -3.3979, -3.2857]],
              [[-2.9329, -2.8003, -2.7369], [-3.0564, -2.4780, -2.0207], [-2.6889, -1.9298, -1.7640]],
          ]      ,  device=_lowerCamelCase      ,  )

      self.assertTrue(torch.allclose(logits[0, :3, :3, :3]      ,  _lowerCamelCase      ,  atol=1E-4   )   )




     @slow
     def  lowerCamelCase_						(							self :Tuple   ):







      '''simple docstring'''



      UpperCamelCase_     :					Union[str, Any]   	=MobileViTVaForSemanticSegmentation.from_pretrained('shehan97/mobilevitv2-1.0-voc-deeplabv3'   )
      UpperCamelCase_     :					Optional[int]   	=model.to(_lowerCamelCase   )

      UpperCamelCase_     :					str   	=MobileViTImageProcessor.from_pretrained('shehan97/mobilevitv2-1.0-voc-deeplabv3'   )

      UpperCamelCase_     :					List[str]   	=prepare_img()
      UpperCamelCase_     :					List[str]   	=image_processor(images=_lowerCamelCase      ,  return_tensors='pt'   ).to(_lowerCamelCase   )

      # forward pass
      with torch.no_grad():
       UpperCamelCase_     :					int   	=model(**_lowerCamelCase   )

      UpperCamelCase_     :					str   	=outputs.logits.detach().cpu()

      UpperCamelCase_     :					Optional[Any]   	=image_processor.post_process_semantic_segmentation(outputs=_lowerCamelCase      ,  target_sizes=[(50, 60)]   )
      UpperCamelCase_     :					Dict   	=torch.Size((50, 60)   )
      self.assertEqual(segmentation[0].shape      ,  _lowerCamelCase   )

      UpperCamelCase_     :					Optional[Any]   	=image_processor.post_process_semantic_segmentation(outputs=_lowerCamelCase   )
      UpperCamelCase_     :					Any   	=torch.Size((32, 32)   )
      self.assertEqual(segmentation[0].shape      ,  _lowerCamelCase   )


"""simple docstring"""






import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


__SCREAMING_SNAKE_CASE							=     datasets.utils.logging.get_logger(__name__)





@dataclass
class 				a__					(						datasets.BuilderConfig       ):
     UpperCAmelCase__         =      None
     UpperCAmelCase__         =      "utf-8"
     UpperCAmelCase__         =      None
     UpperCAmelCase__         =      None
     UpperCAmelCase__         =      True  # deprecated
     UpperCAmelCase__         =      None  # deprecated
     UpperCAmelCase__         =      10 << 20  # 10MB
     UpperCAmelCase__         =      None





class 				a__					(						datasets.ArrowBasedBuilder       ):
     UpperCAmelCase__         =      JsonConfig



     def  lowerCamelCase_						(							self :List[str]   ):







      '''simple docstring'''



      if self.config.block_size is not None:
       logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'   )
       UpperCamelCase_     :					List[Any]   	=self.config.block_size
      if self.config.use_threads is not True:
       logger.warning(
           'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'   )
      if self.config.newlines_in_values is not None:
       raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'   )
      return datasets.DatasetInfo(features=self.config.features   )



     def  lowerCamelCase_						(							self :Tuple      ,  _lowerCamelCase :List[str]   ):







      '''simple docstring'''



      if not self.config.data_files:
       raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''   )
      UpperCamelCase_     :					int   	=dl_manager.download_and_extract(self.config.data_files   )
      if isinstance(_lowerCamelCase      ,  (str, list, tuple)   ):
       UpperCamelCase_     :					Tuple   	=data_files
       if isinstance(_lowerCamelCase      ,  _lowerCamelCase   ):
        UpperCamelCase_     :					Union[str, Any]   	=[files]
       UpperCamelCase_     :					List[str]   	=[dl_manager.iter_files(_lowerCamelCase   ) for file in files]
       return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,  gen_kwargs={'files': files}   )]
      UpperCamelCase_     :					Optional[Any]   	=[]
      for split_name, files in data_files.items():
       if isinstance(_lowerCamelCase      ,  _lowerCamelCase   ):
        UpperCamelCase_     :					Tuple   	=[files]
       UpperCamelCase_     :					Optional[Any]   	=[dl_manager.iter_files(_lowerCamelCase   ) for file in files]
       splits.append(datasets.SplitGenerator(name=_lowerCamelCase      ,  gen_kwargs={'files': files}   )   )
      return splits



     def  lowerCamelCase_						(							self :int      ,  _lowerCamelCase :pa.Table   ):







      '''simple docstring'''



      if self.config.features is not None:
       # adding missing columns
       for column_name in set(self.config.features   ) - set(pa_table.column_names   ):
        UpperCamelCase_     :					Dict   	=self.config.features.arrow_schema.field(_lowerCamelCase   ).type
        UpperCamelCase_     :					Optional[Any]   	=pa_table.append_column(_lowerCamelCase      ,  pa.array([None] * len(_lowerCamelCase   )      ,  type=_lowerCamelCase   )   )
       # more expensive cast to support nested structures with keys in a different order
       # allows str <-> int/float or str to Audio for example
       UpperCamelCase_     :					int   	=table_cast(_lowerCamelCase      ,  self.config.features.arrow_schema   )
      return pa_table




     def  lowerCamelCase_						(							self :Dict      ,  _lowerCamelCase :Optional[int]   ):







      '''simple docstring'''



      for file_idx, file in enumerate(itertools.chain.from_iterable(_lowerCamelCase   )   ):
       # If the file is one json object and if we need to look at the list of items in one specific field
       if self.config.field is not None:
        with open(_lowerCamelCase      ,  encoding=self.config.encoding      ,  errors=self.config.encoding_errors   ) as f:
         UpperCamelCase_     :					List[str]   	=json.load(_lowerCamelCase   )

        # We keep only the field we are interested in
        UpperCamelCase_     :					Tuple   	=dataset[self.config.field]

        # We accept two format: a list of dicts or a dict of lists
        if isinstance(_lowerCamelCase      ,  (list, tuple)   ):
         UpperCamelCase_     :					Tuple   	=set().union(*[row.keys() for row in dataset]   )
         UpperCamelCase_     :					str   	={col: [row.get(_lowerCamelCase   ) for row in dataset] for col in keys}
        else:
         UpperCamelCase_     :					Any   	=dataset
        UpperCamelCase_     :					List[Any]   	=pa.Table.from_pydict(_lowerCamelCase   )
        yield file_idx, self._cast_table(_lowerCamelCase   )

       # If the file has one json object per line
       else:
        with open(_lowerCamelCase      ,  'rb'   ) as f:
         UpperCamelCase_     :					Any   	=0
         # Use block_size equal to the chunk size divided by 32 to leverage multithreading
         # Set a default minimum value of 16kB if the chunk size is really small
         UpperCamelCase_     :					List[str]   	=max(self.config.chunksize // 32      ,  16 << 10   )
         UpperCamelCase_     :					Dict   	=(
             self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
         )
         while True:
          UpperCamelCase_     :					Optional[Any]   	=f.read(self.config.chunksize   )
          if not batch:
           break
          # Finish current line
          try:
           batch += f.readline()
          except (AttributeError, io.UnsupportedOperation):
           batch += readline(_lowerCamelCase   )
          # PyArrow only accepts utf-8 encoded bytes
          if self.config.encoding != "utf-8":
           UpperCamelCase_     :					List[Any]   	=batch.decode(self.config.encoding      ,  errors=_lowerCamelCase   ).encode('utf-8'   )
          try:
           while True:
            try:
             UpperCamelCase_     :					str   	=paj.read_json(
                 io.BytesIO(_lowerCamelCase   )      ,  read_options=paj.ReadOptions(block_size=_lowerCamelCase   )   )
             break
            except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
             if (
                 isinstance(_lowerCamelCase      ,  pa.ArrowInvalid   )
                 and "straddling" not in str(_lowerCamelCase   )
                 or block_size > len(_lowerCamelCase   )
             ):
              raise
             else:
              # Increase the block size in case it was too small.
              # The block size will be reset for the next file.
              logger.debug(
                  f'''Batch of {len(_lowerCamelCase   )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''   )
              block_size *= 2
          except pa.ArrowInvalid as e:
           try:
            with open(
                _lowerCamelCase      ,  encoding=self.config.encoding      ,  errors=self.config.encoding_errors   ) as f:
             UpperCamelCase_     :					List[Any]   	=json.load(_lowerCamelCase   )
           except json.JSONDecodeError:
            logger.error(f'''Failed to read file \'{file}\' with error {type(_lowerCamelCase   )}: {e}'''   )
            raise e
           # If possible, parse the file as a list of json objects and exit the loop
           if isinstance(_lowerCamelCase      ,  _lowerCamelCase   ):  # list is the only sequence type supported in JSON
            try:
             UpperCamelCase_     :					str   	=set().union(*[row.keys() for row in dataset]   )
             UpperCamelCase_     :					List[Any]   	={col: [row.get(_lowerCamelCase   ) for row in dataset] for col in keys}
             UpperCamelCase_     :					Optional[Any]   	=pa.Table.from_pydict(_lowerCamelCase   )
            except (pa.ArrowInvalid, AttributeError) as e:
             logger.error(f'''Failed to read file \'{file}\' with error {type(_lowerCamelCase   )}: {e}'''   )
             raise ValueError(f'''Not able to read records in the JSON file at {file}.'''   ) from None
            yield file_idx, self._cast_table(_lowerCamelCase   )
            break
           else:
            logger.error(f'''Failed to read file \'{file}\' with error {type(_lowerCamelCase   )}: {e}'''   )
            raise ValueError(
                f'''Not able to read records in the JSON file at {file}. '''
                f'''You should probably indicate the field of the JSON file containing your records. '''
                f'''This JSON file contain the following fields: {str(list(dataset.keys()   )   )}. '''
                f'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''   ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
          yield (file_idx, batch_idx), self._cast_table(_lowerCamelCase   )
          batch_idx += 1
