




"""simple docstring"""
from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import (
					    TFConvBertForMaskedLM,
					    TFConvBertForMultipleChoice,
					    TFConvBertForQuestionAnswering,
					    TFConvBertForSequenceClassification,
					    TFConvBertForTokenClassification,
					    TFConvBertModel,
					)

class __lowercase:




					'''simple docstring'''


					def __init__(    self						,	__a						,	__a=13						,	__a=7						,	__a=True						,	__a=True						,	__a=True						,	__a=True						,	__a=99						,	__a=32						,	__a=2						,	__a=4						,	__a=37						,	__a="gelu"						,	__a=0.1						,	__a=0.1						,	__a=512						,	__a=16						,	__a=2						,	__a=0.02						,	__a=3						,	__a=4						,	__a=None						,	):
								__lowerCamelCase					:	Tuple	       =				parent
								__lowerCamelCase					:	List[str]	       =				13
								__lowerCamelCase					:	Union[str, Any]	       =				7
								__lowerCamelCase					:	Dict	       =				True
								__lowerCamelCase					:	str	       =				True
								__lowerCamelCase					:	Tuple	       =				True
								__lowerCamelCase					:	Union[str, Any]	       =				True
								__lowerCamelCase					:	Optional[int]	       =				99
								__lowerCamelCase					:	Dict	       =				384
								__lowerCamelCase					:	Dict	       =				2
								__lowerCamelCase					:	str	       =				4
								__lowerCamelCase					:	Optional[Any]	       =				37
								__lowerCamelCase					:	Tuple	       =				'gelu'
								__lowerCamelCase					:	Union[str, Any]	       =				0.1
								__lowerCamelCase					:	Union[str, Any]	       =				0.1
								__lowerCamelCase					:	Tuple	       =				512
								__lowerCamelCase					:	str	       =				16
								__lowerCamelCase					:	int	       =				2
								__lowerCamelCase					:	int	       =				0.02
								__lowerCamelCase					:	List[str]	       =				3
								__lowerCamelCase					:	Optional[Any]	       =				4
								__lowerCamelCase					:	List[Any]	       =				128
								__lowerCamelCase					:	List[str]	       =				2
								__lowerCamelCase					:	List[Any]	       =				9
								__lowerCamelCase					:	str	       =				1
								__lowerCamelCase					:	List[str]	       =				None
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Optional[Any]	       =				ids_tensor([self.batch_size, self.seq_length]						,	self.vocab_size     )

								__lowerCamelCase					:	Tuple	       =				None
								if self.use_input_mask:
											__lowerCamelCase					:	Optional[int]	       =				random_attention_mask([self.batch_size, self.seq_length]     )

								__lowerCamelCase					:	Optional[Any]	       =				None
								if self.use_token_type_ids:
											__lowerCamelCase					:	Optional[Any]	       =				ids_tensor([self.batch_size, self.seq_length]						,	self.type_vocab_size     )

								__lowerCamelCase					:	Tuple	       =				None
								__lowerCamelCase					:	int	       =				None
								__lowerCamelCase					:	Tuple	       =				None
								if self.use_labels:
											__lowerCamelCase					:	Tuple	       =				ids_tensor([self.batch_size]						,	self.type_sequence_label_size     )
											__lowerCamelCase					:	Tuple	       =				ids_tensor([self.batch_size, self.seq_length]						,	self.num_labels     )
											__lowerCamelCase					:	Tuple	       =				ids_tensor([self.batch_size]						,	self.num_choices     )

								__lowerCamelCase					:	List[Any]	       =				ConvBertConfig(
								    vocab_size=self.vocab_size						,	hidden_size=self.hidden_size						,	num_hidden_layers=self.num_hidden_layers						,	num_attention_heads=self.num_attention_heads						,	intermediate_size=self.intermediate_size						,	hidden_act=self.hidden_act						,	hidden_dropout_prob=self.hidden_dropout_prob						,	attention_probs_dropout_prob=self.attention_probs_dropout_prob						,	max_position_embeddings=self.max_position_embeddings						,	type_vocab_size=self.type_vocab_size						,	initializer_range=self.initializer_range						,	return_dict=__a						,	)

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
					def     snake_case_				(    self						,	__a						,	__a						,	__a						,	__a						,	__a						,	__a						,	__a     ):
								__lowerCamelCase					:	Union[str, Any]	       =				TFConvBertModel(config=__a     )
								__lowerCamelCase					:	List[Any]	       =				{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

								__lowerCamelCase					:	List[Any]	       =				[input_ids, input_mask]
								__lowerCamelCase					:	int	       =				model(__a     )

								__lowerCamelCase					:	List[Any]	       =				model(__a     )

								self.parent.assertEqual(result.last_hidden_state.shape						,	(self.batch_size, self.seq_length, self.hidden_size)     )
					def     snake_case_				(    self						,	__a						,	__a						,	__a						,	__a						,	__a						,	__a						,	__a     ):
								__lowerCamelCase					:	List[Any]	       =				TFConvBertForMaskedLM(config=__a     )
								__lowerCamelCase					:	List[str]	       =				{
								    'input_ids': input_ids,
								    'attention_mask': input_mask,
								    'token_type_ids': token_type_ids,
								}
								__lowerCamelCase					:	str	       =				model(__a     )
								self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.seq_length, self.vocab_size)     )
					def     snake_case_				(    self						,	__a						,	__a						,	__a						,	__a						,	__a						,	__a						,	__a     ):
								__lowerCamelCase					:	Any	       =				self.num_labels
								__lowerCamelCase					:	List[Any]	       =				TFConvBertForSequenceClassification(config=__a     )
								__lowerCamelCase					:	Any	       =				{
								    'input_ids': input_ids,
								    'attention_mask': input_mask,
								    'token_type_ids': token_type_ids,
								}

								__lowerCamelCase					:	Optional[int]	       =				model(__a     )
								self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.num_labels)     )
					def     snake_case_				(    self						,	__a						,	__a						,	__a						,	__a						,	__a						,	__a						,	__a     ):
								__lowerCamelCase					:	List[str]	       =				self.num_choices
								__lowerCamelCase					:	Union[str, Any]	       =				TFConvBertForMultipleChoice(config=__a     )
								__lowerCamelCase					:	Dict	       =				tf.tile(tf.expand_dims(__a						,	1     )						,	(1, self.num_choices, 1)     )
								__lowerCamelCase					:	Optional[Any]	       =				tf.tile(tf.expand_dims(__a						,	1     )						,	(1, self.num_choices, 1)     )
								__lowerCamelCase					:	Union[str, Any]	       =				tf.tile(tf.expand_dims(__a						,	1     )						,	(1, self.num_choices, 1)     )
								__lowerCamelCase					:	List[Any]	       =				{
								    'input_ids': multiple_choice_inputs_ids,
								    'attention_mask': multiple_choice_input_mask,
								    'token_type_ids': multiple_choice_token_type_ids,
								}
								__lowerCamelCase					:	List[str]	       =				model(__a     )
								self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.num_choices)     )
					def     snake_case_				(    self						,	__a						,	__a						,	__a						,	__a						,	__a						,	__a						,	__a     ):
								__lowerCamelCase					:	Union[str, Any]	       =				self.num_labels
								__lowerCamelCase					:	List[Any]	       =				TFConvBertForTokenClassification(config=__a     )
								__lowerCamelCase					:	Tuple	       =				{
								    'input_ids': input_ids,
								    'attention_mask': input_mask,
								    'token_type_ids': token_type_ids,
								}
								__lowerCamelCase					:	str	       =				model(__a     )
								self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.seq_length, self.num_labels)     )
					def     snake_case_				(    self						,	__a						,	__a						,	__a						,	__a						,	__a						,	__a						,	__a     ):
								__lowerCamelCase					:	Dict	       =				TFConvBertForQuestionAnswering(config=__a     )
								__lowerCamelCase					:	List[Any]	       =				{
								    'input_ids': input_ids,
								    'attention_mask': input_mask,
								    'token_type_ids': token_type_ids,
								}

								__lowerCamelCase					:	Tuple	       =				model(__a     )
								self.parent.assertEqual(result.start_logits.shape						,	(self.batch_size, self.seq_length)     )
								self.parent.assertEqual(result.end_logits.shape						,	(self.batch_size, self.seq_length)     )





					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Dict	       =				self.prepare_config_and_inputs()
								(
								    __lowerCamelCase      
								)					:	Dict	       =				config_and_inputs
								__lowerCamelCase					:	Union[str, Any]	       =				{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
								return config, inputs_dict



@require_tf
class __lowercase(      lowercase__	,	lowercase__	,	unittest.TestCase				):




					'''simple docstring'''


					__a						:						Dict     		= (
					    (
					        TFConvBertModel,
					        TFConvBertForMaskedLM,
					        TFConvBertForQuestionAnswering,
					        TFConvBertForSequenceClassification,
					        TFConvBertForTokenClassification,
					        TFConvBertForMultipleChoice,
					    )
					    if is_tf_available()
					    else ()
					)
					__a						:						Union[str, Any]     		= (
					    {
					        'feature-extraction': TFConvBertModel,
					        'fill-mask': TFConvBertForMaskedLM,
					        'question-answering': TFConvBertForQuestionAnswering,
					        'text-classification': TFConvBertForSequenceClassification,
					        'token-classification': TFConvBertForTokenClassification,
					        'zero-shot': TFConvBertForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					__a						:						Dict     		= False
					__a						:						Optional[int]     		= False
					__a						:						Optional[Any]     		= False
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Dict	       =				TFConvBertModelTester(self     )
								__lowerCamelCase					:	Optional[int]	       =				ConfigTester(self						,	config_class=__a						,	hidden_size=37     )
					def     snake_case_				(    self     ):
								self.config_tester.run_common_tests()
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Optional[int]	       =				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*__a     )
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Any	       =				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*__a     )
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Any	       =				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_multiple_choice(*__a     )
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	List[Any]	       =				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*__a     )
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	List[str]	       =				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*__a     )
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Any	       =				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*__a     )
					@slow
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Tuple	       =				self.model_tester.prepare_config_and_inputs_for_common()
								__lowerCamelCase					:	Tuple	       =				True
								__lowerCamelCase					:	Any	       =				True

								if hasattr(__a						,	'use_cache'     ):
											__lowerCamelCase					:	int	       =				True

								__lowerCamelCase					:	Tuple	       =				getattr(self.model_tester						,	'encoder_seq_length'						,	self.model_tester.seq_length     )
								__lowerCamelCase					:	Any	       =				getattr(self.model_tester						,	'key_length'						,	__a     )

								for model_class in self.all_model_classes:
											__lowerCamelCase					:	Optional[Any]	       =				self._prepare_for_class(__a						,	__a     )
											__lowerCamelCase					:	int	       =				model_class(__a     )
											__lowerCamelCase					:	Tuple	       =				len(model(__a     )     )

											with tempfile.TemporaryDirectory() as tmpdirname:
														model.save_pretrained(__a						,	saved_model=__a     )
														__lowerCamelCase					:	Dict	       =				os.path.join(__a						,	'saved_model'						,	'1'     )
														__lowerCamelCase					:	Optional[Any]	       =				tf.keras.models.load_model(__a     )
														__lowerCamelCase					:	str	       =				model(__a     )

														if self.is_encoder_decoder:
																	__lowerCamelCase					:	Optional[Any]	       =				outputs['encoder_hidden_states']
																	__lowerCamelCase					:	Tuple	       =				outputs['encoder_attentions']
														else:
																	__lowerCamelCase					:	Union[str, Any]	       =				outputs['hidden_states']
																	__lowerCamelCase					:	Union[str, Any]	       =				outputs['attentions']

														self.assertEqual(len(__a     )						,	__a     )

														__lowerCamelCase					:	Union[str, Any]	       =				getattr(
														    self.model_tester						,	'expected_num_hidden_layers'						,	self.model_tester.num_hidden_layers + 1     )

														self.assertEqual(len(__a     )						,	__a     )
														self.assertListEqual(
														    list(output_hidden_states[0].shape[-2:]     )						,	[self.model_tester.seq_length, self.model_tester.hidden_size]						,	)

														self.assertEqual(len(__a     )						,	self.model_tester.num_hidden_layers     )
														self.assertListEqual(
														    list(output_attentions[0].shape[-3:]     )						,	[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]						,	)
					@slow
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	List[Any]	       =				TFConvBertModel.from_pretrained('YituTech/conv-bert-base'     )
								self.assertIsNotNone(__a     )





					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Optional[int]	       =				self.model_tester.prepare_config_and_inputs_for_common()
								__lowerCamelCase					:	List[Any]	       =				True
								__lowerCamelCase					:	Union[str, Any]	       =				getattr(self.model_tester						,	'decoder_seq_length'						,	self.model_tester.seq_length     )
								__lowerCamelCase					:	str	       =				getattr(self.model_tester						,	'encoder_seq_length'						,	self.model_tester.seq_length     )
								__lowerCamelCase					:	int	       =				getattr(self.model_tester						,	'key_length'						,	__a     )
								__lowerCamelCase					:	List[Any]	       =				getattr(self.model_tester						,	'key_length'						,	__a     )

								def check_decoder_attentions_output(__a     ):
											__lowerCamelCase					:	str	       =				len(__a     )
											self.assertEqual(out_len % 2						,	0     )
											__lowerCamelCase					:	str	       =				outputs.decoder_attentions
											self.assertEqual(len(__a     )						,	self.model_tester.num_hidden_layers     )
											self.assertListEqual(
											    list(decoder_attentions[0].shape[-3:]     )						,	[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]						,	)

								def check_encoder_attentions_output(__a     ):
											__lowerCamelCase					:	Dict	       =				[
											    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
											]
											self.assertEqual(len(__a     )						,	self.model_tester.num_hidden_layers     )
											self.assertListEqual(
											    list(attentions[0].shape[-3:]     )						,	[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]						,	)

								for model_class in self.all_model_classes:
											__lowerCamelCase					:	Tuple	       =				True
											__lowerCamelCase					:	Optional[Any]	       =				False
											__lowerCamelCase					:	Optional[Any]	       =				model_class(__a     )
											__lowerCamelCase					:	Any	       =				model(self._prepare_for_class(__a						,	__a     )     )
											__lowerCamelCase					:	Optional[int]	       =				len(__a     )
											self.assertEqual(config.output_hidden_states						,	__a     )
											check_encoder_attentions_output(__a     )

											if self.is_encoder_decoder:
														__lowerCamelCase					:	Any	       =				model_class(__a     )
														__lowerCamelCase					:	Dict	       =				model(self._prepare_for_class(__a						,	__a     )     )
														self.assertEqual(config.output_hidden_states						,	__a     )
														check_decoder_attentions_output(__a     )

											# Check that output attentions can also be changed via the config
											del inputs_dict["output_attentions"]
											__lowerCamelCase					:	str	       =				True
											__lowerCamelCase					:	Any	       =				model_class(__a     )
											__lowerCamelCase					:	List[str]	       =				model(self._prepare_for_class(__a						,	__a     )     )
											self.assertEqual(config.output_hidden_states						,	__a     )
											check_encoder_attentions_output(__a     )

											# Check attention is always last and order is fine
											__lowerCamelCase					:	Any	       =				True
											__lowerCamelCase					:	Optional[int]	       =				True
											__lowerCamelCase					:	Any	       =				model_class(__a     )
											__lowerCamelCase					:	List[Any]	       =				model(self._prepare_for_class(__a						,	__a     )     )

											self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)						,	len(__a     )     )
											self.assertEqual(model.config.output_hidden_states						,	__a     )
											check_encoder_attentions_output(__a     )



@require_tf
class __lowercase(      unittest.TestCase				):




					'''simple docstring'''


					@slow
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Tuple	       =				TFConvBertModel.from_pretrained('YituTech/conv-bert-base'     )
								__lowerCamelCase					:	List[str]	       =				tf.constant([[0, 1, 2, 3, 4, 5]]     )
								__lowerCamelCase					:	Optional[int]	       =				model(__a     )[0]

								__lowerCamelCase					:	int	       =				[1, 6, 768]
								self.assertEqual(output.shape						,	__a     )

								__lowerCamelCase					:	str	       =				tf.constant(
								    [
								        [
								            [-0.03_475_493, -0.4_686_034, -0.30_638_832],
								            [0.22_637_248, -0.26_988_646, -0.7_423_424],
								            [0.10_324_868, -0.45_013_508, -0.58_280_784],
								        ]
								    ]     )
								tf.debugging.assert_near(output[:, :3, :3]						,	__a						,	atol=1E-4     )








"""simple docstring"""
import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

class __lowercase(      lowercase__				):




					'''simple docstring'''


					def __init__(    self						,	__a						,	__a = None						,	__a = None						,	__a = None						,	__a = False						,	__a = False						,	__a = None						,	__a = None						,	**__a						,	):
								super().__init__(
								    __a						,	split=__a						,	features=__a						,	cache_dir=__a						,	keep_in_memory=__a						,	streaming=__a						,	num_proc=__a						,	**__a						,	)
								__lowerCamelCase					:	List[Any]	       =				field
								__lowerCamelCase					:	Dict	       =				path_or_paths if isinstance(__a						,	__a     ) else {self.split: path_or_paths}
								__lowerCamelCase					:	str	       =				Json(
								    cache_dir=__a						,	data_files=__a						,	features=__a						,	field=__a						,	**__a						,	)





					def     snake_case_				(    self     ):
								# Build iterable dataset
								if self.streaming:
											__lowerCamelCase					:	str	       =				self.builder.as_streaming_dataset(split=self.split     )
								# Build regular (map-style) dataset
								else:
											__lowerCamelCase					:	Any	       =				None
											__lowerCamelCase					:	Any	       =				None
											__lowerCamelCase					:	str	       =				None
											__lowerCamelCase					:	int	       =				None

											self.builder.download_and_prepare(
											    download_config=__a						,	download_mode=__a						,	verification_mode=__a						,	base_path=__a						,	num_proc=self.num_proc						,	)
											__lowerCamelCase					:	List[Any]	       =				self.builder.as_dataset(
											    split=self.split						,	verification_mode=__a						,	in_memory=self.keep_in_memory     )
								return dataset



class __lowercase:




					'''simple docstring'''


					def __init__(    self						,	__a						,	__a						,	__a = None						,	__a = None						,	**__a						,	):
								if num_proc is not None and num_proc <= 0:
											raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''     )

								__lowerCamelCase					:	Optional[int]	       =				dataset
								__lowerCamelCase					:	Optional[int]	       =				path_or_buf
								__lowerCamelCase					:	List[str]	       =				batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
								__lowerCamelCase					:	str	       =				num_proc
								__lowerCamelCase					:	Optional[Any]	       =				'utf-8'
								__lowerCamelCase					:	Optional[int]	       =				to_json_kwargs
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Tuple	       =				self.to_json_kwargs.pop('path_or_buf'						,	__a     )
								__lowerCamelCase					:	Any	       =				self.to_json_kwargs.pop('orient'						,	'records'     )
								__lowerCamelCase					:	List[Any]	       =				self.to_json_kwargs.pop('lines'						,	True if orient == 'records' else False     )
								__lowerCamelCase					:	Tuple	       =				self.to_json_kwargs.pop('index'						,	False if orient in ['split', 'table'] else True     )
								__lowerCamelCase					:	int	       =				self.to_json_kwargs.pop('compression'						,	__a     )

								if compression not in [None, "infer", "gzip", "bz2", "xz"]:
											raise NotImplementedError(f'''`datasets` currently does not support {compression} compression'''     )

								if isinstance(self.path_or_buf						,	(str, bytes, os.PathLike)     ):
											with fsspec.open(self.path_or_buf						,	'wb'						,	compression=__a     ) as buffer:
														__lowerCamelCase					:	int	       =				self._write(file_obj=__a						,	orient=__a						,	lines=__a						,	index=__a						,	**self.to_json_kwargs     )
								else:
											if compression:
														raise NotImplementedError(
														    f'''The compression parameter is not supported when writing to a buffer, but compression={compression}'''
														    ' was passed. Please provide a local path instead.'     )
											__lowerCamelCase					:	Dict	       =				self._write(
											    file_obj=self.path_or_buf						,	orient=__a						,	lines=__a						,	index=__a						,	**self.to_json_kwargs     )
								return written
					def     snake_case_				(    self						,	__a     ):
								__lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase					:	Union[str, Any]	       =				args

								__lowerCamelCase					:	int	       =				query_table(
								    table=self.dataset.data						,	key=slice(__a						,	offset + self.batch_size     )						,	indices=self.dataset._indices						,	)
								__lowerCamelCase					:	List[str]	       =				batch.to_pandas().to_json(
								    path_or_buf=__a						,	orient=__a						,	lines=__a						,	index=__a						,	**__a     )
								if not json_str.endswith('\n'     ):
											json_str += "\n"
								return json_str.encode(self.encoding     )





					def     snake_case_				(    self						,	__a						,	__a						,	__a						,	__a						,	**__a						,	):
								__lowerCamelCase					:	Any	       =				0

								if self.num_proc is None or self.num_proc == 1:
											for offset in logging.tqdm(
											    range(0						,	len(self.dataset     )						,	self.batch_size     )						,	unit='ba'						,	disable=not logging.is_progress_bar_enabled()						,	desc='Creating json from Arrow format'						,	):
														__lowerCamelCase					:	List[str]	       =				self._batch_json((offset, orient, lines, index, to_json_kwargs)     )
														written += file_obj.write(__a     )
								else:
											__lowerCamelCase       ,      __lowerCamelCase					:	Optional[int]	       =				len(self.dataset     ), self.batch_size
											with multiprocessing.Pool(self.num_proc     ) as pool:
														for json_str in logging.tqdm(
														    pool.imap(
														        self._batch_json						,	[(offset, orient, lines, index, to_json_kwargs) for offset in range(0						,	__a						,	__a     )]						,	)						,	total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size						,	unit='ba'						,	disable=not logging.is_progress_bar_enabled()						,	desc='Creating json from Arrow format'						,	):
																	written += file_obj.write(__a     )

								return written



