



'''simple docstring'''


import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig


def  _lowerCamelCase   (				lowerCamelCase_ :							Union[str, Any]     ,					lowerCamelCase_ :							Optional[int]     ,					lowerCamelCase_ :							Optional[int]=0       ):



     """simple docstring"""





     if name is None:
          UpperCAmelCase_							:       Tuple       =				None
     else:
          UpperCAmelCase_							:       Union[str, Any]       =				'.' * max(0     ,					spaces - 2       ) + '# {:' + str(50 - spaces       ) + 's}'
          UpperCAmelCase_							:       Dict       =				fmt.format(lowerCamelCase_       )

     # Print and recurse (if needed).
     if isinstance(lowerCamelCase_     ,					lowerCamelCase_       ):
          if msg is not None:
               print(lowerCamelCase_       )
          for k in val.keys():
               recursive_print(lowerCamelCase_     ,					val[k]     ,					spaces + 2       )
     elif isinstance(lowerCamelCase_     ,					torch.Tensor       ):
          print(lowerCamelCase_     ,					':'     ,					val.size()       )
     else:
          print(lowerCamelCase_     ,					':'     ,					lowerCamelCase_       )


def  _lowerCamelCase   (				lowerCamelCase_ :							Dict     ,					lowerCamelCase_ :							List[Any]     ,					lowerCamelCase_ :							Any     ,					lowerCamelCase_ :							int     ,					lowerCamelCase_ :							List[Any]       ):



     """simple docstring"""





     UpperCAmelCase_							:       List[str]       =				param.size()
     if checkpoint_version == 1.0:
          # version 1.0 stores [num_heads * hidden_size * num_splits, :]
          UpperCAmelCase_							:       Tuple       =				(num_heads, hidden_size, num_splits) + input_shape[1:]
          UpperCAmelCase_							:       List[str]       =				param.view(*lowerCamelCase_       )
          UpperCAmelCase_							:       Union[str, Any]       =				param.transpose(0     ,					2       )
          UpperCAmelCase_							:       Optional[int]       =				param.transpose(1     ,					2       ).contiguous()
     elif checkpoint_version >= 2.0:
          # other versions store [num_heads * num_splits * hidden_size, :]
          UpperCAmelCase_							:       Union[str, Any]       =				(num_heads, num_splits, hidden_size) + input_shape[1:]
          UpperCAmelCase_							:       Any       =				param.view(*lowerCamelCase_       )
          UpperCAmelCase_							:       Dict       =				param.transpose(0     ,					1       ).contiguous()
     UpperCAmelCase_							:       Optional[int]       =				param.view(*lowerCamelCase_       )
     return param


def  _lowerCamelCase   (				lowerCamelCase_ :							Any     ,					lowerCamelCase_ :							str     ,					lowerCamelCase_ :							Any       ):



     """simple docstring"""





     UpperCAmelCase_							:       Optional[int]       =				{}

     # old versions did not store training args
     UpperCAmelCase_							:       Any       =				input_state_dict.get('args'     ,					lowerCamelCase_       )
     if ds_args is not None:
          # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
          # from pprint import pprint
          # pprint(vars(ds_args))

          UpperCAmelCase_							:       Optional[Any]       =				ds_args.padded_vocab_size
          UpperCAmelCase_							:       Optional[Any]       =				ds_args.max_position_embeddings
          UpperCAmelCase_							:       Tuple       =				ds_args.hidden_size
          UpperCAmelCase_							:       Optional[Any]       =				ds_args.num_layers
          UpperCAmelCase_							:       int       =				ds_args.num_attention_heads
          UpperCAmelCase_							:       int       =				ds_args.ffn_hidden_size
          # pprint(config)

     # The number of heads.
     UpperCAmelCase_							:       Optional[int]       =				config.n_head
     # The hidden_size per head.
     UpperCAmelCase_							:       Dict       =				config.n_embd // config.n_head
     # Megatron-LM checkpoint version
     if "checkpoint_version" in input_state_dict.keys():
          UpperCAmelCase_							:       List[Any]       =				input_state_dict['checkpoint_version']
     else:
          UpperCAmelCase_							:       Optional[int]       =				0.0

     # The model.
     UpperCAmelCase_							:       Union[str, Any]       =				input_state_dict['model']
     # The language model.
     UpperCAmelCase_							:       Optional[Any]       =				model['language_model']
     # The embeddings.
     UpperCAmelCase_							:       str       =				lm['embedding']

     # The word embeddings.
     UpperCAmelCase_							:       str       =				embeddings['word_embeddings']['weight']
     # Truncate the embedding table to vocab_size rows.
     UpperCAmelCase_							:       Any       =				word_embeddings[: config.vocab_size, :]
     UpperCAmelCase_							:       Tuple       =				word_embeddings

     # The position embeddings.
     UpperCAmelCase_							:       Tuple       =				embeddings['position_embeddings']['weight']
     # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
     UpperCAmelCase_							:       Dict       =				pos_embeddings.size(0       )
     if n_positions != config.n_positions:
          raise ValueError(
              F'''pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'''       )
     # Store the position embeddings.
     UpperCAmelCase_							:       Any       =				pos_embeddings

     # The transformer.
     UpperCAmelCase_							:       Dict       =				lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

     # The regex to extract layer names.
     UpperCAmelCase_							:       Optional[int]       =				re.compile(R'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'       )

     # The simple map of names for "automated" rules.
     UpperCAmelCase_							:       str       =				{
         'attention.dense': '.attn.c_proj.',
         'self_attention.dense': '.attn.c_proj.',
         'mlp.dense_h_to_4h': '.mlp.c_fc.',
         'mlp.dense_4h_to_h': '.mlp.c_proj.',
     }

     # Extract the layers.
     for key, val in transformer.items():
          # Match the name.
          UpperCAmelCase_							:       Tuple       =				layer_re.match(lowerCamelCase_       )

          # Stop if that's not a layer
          if m is None:
               break

          # The index of the layer.
          UpperCAmelCase_							:       List[str]       =				int(m.group(1       )       )
          # The name of the operation.
          UpperCAmelCase_							:       Any       =				m.group(2       )
          # Is it a weight or a bias?
          UpperCAmelCase_							:       Optional[int]       =				m.group(3       )

          # The name of the layer.
          UpperCAmelCase_							:       int       =				F'''transformer.h.{layer_idx}'''

          # For layernorm(s), simply store the layer norm.
          if op_name.endswith('layernorm'       ):
               UpperCAmelCase_							:       str       =				'ln_1' if op_name.startswith('input'       ) else 'ln_2'
               UpperCAmelCase_							:       Optional[int]       =				val

          # Transpose the QKV matrix.
          elif (
              op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
          ) and weight_or_bias == "weight":
               # Insert a tensor of 1x1xDxD bias.
               UpperCAmelCase_							:       str       =				torch.tril(torch.ones((n_positions, n_positions)     ,					dtype=torch.floataa       )       ).view(
                   1     ,					1     ,					lowerCamelCase_     ,					lowerCamelCase_       )
               UpperCAmelCase_							:       int       =				causal_mask

               # Insert a "dummy" tensor for masked_bias.
               UpperCAmelCase_							:       List[Any]       =				torch.tensor(-1e4     ,					dtype=torch.floataa       )
               UpperCAmelCase_							:       int       =				masked_bias

               UpperCAmelCase_							:       List[Any]       =				fix_query_key_value_ordering(lowerCamelCase_     ,					lowerCamelCase_     ,					3     ,					lowerCamelCase_     ,					lowerCamelCase_       )
               # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
               UpperCAmelCase_							:       str       =				out_val.transpose(0     ,					1       ).contiguous()
               # Store.
               UpperCAmelCase_							:       Dict       =				out_val

          # Transpose the bias.
          elif (
              op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
          ) and weight_or_bias == "bias":
               UpperCAmelCase_							:       int       =				fix_query_key_value_ordering(lowerCamelCase_     ,					lowerCamelCase_     ,					3     ,					lowerCamelCase_     ,					lowerCamelCase_       )
               # Store. No change of shape.
               UpperCAmelCase_							:       Tuple       =				out_val

          # Transpose the weights.
          elif weight_or_bias == "weight":
               UpperCAmelCase_							:       Dict       =				megatron_to_transformers[op_name]
               UpperCAmelCase_							:       str       =				val.transpose(0     ,					1       )

          # Copy the bias.
          elif weight_or_bias == "bias":
               UpperCAmelCase_							:       Optional[int]       =				megatron_to_transformers[op_name]
               UpperCAmelCase_							:       str       =				val

    # DEBUG.
     assert config.n_layer == layer_idx + 1

     # The final layernorm.
     UpperCAmelCase_							:       Tuple       =				transformer['final_layernorm.weight']
     UpperCAmelCase_							:       Tuple       =				transformer['final_layernorm.bias']

     # For LM head, transformers' wants the matrix to weight embeddings.
     UpperCAmelCase_							:       List[str]       =				word_embeddings

     # It should be done!
     return output_state_dict


def  _lowerCamelCase   (				):



     """simple docstring"""





     UpperCAmelCase_							:       List[str]       =				argparse.ArgumentParser()
     parser.add_argument('--print-checkpoint-structure'     ,					action='store_true'       )
     parser.add_argument(
         'path_to_checkpoint'     ,					type=lowerCamelCase_     ,					help='Path to the checkpoint file (.zip archive or direct .pt file)'     ,					)
     parser.add_argument(
         '--config_file'     ,					default=''     ,					type=lowerCamelCase_     ,					help='An optional config json file describing the pre-trained model.'     ,					)
     UpperCAmelCase_							:       Tuple       =				parser.parse_args()

     # Extract the basename.
     UpperCAmelCase_							:       Optional[Any]       =				os.path.dirname(args.path_to_checkpoint       )

     # Load the model.
     # the .zip is very optional, let's keep it for backward compatibility
     print(F'''Extracting PyTorch state dictionary from {args.path_to_checkpoint}'''       )
     if args.path_to_checkpoint.endswith('.zip'       ):
          with zipfile.ZipFile(args.path_to_checkpoint     ,					'r'       ) as checkpoint:
               with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'       ) as pytorch_dict:
                    UpperCAmelCase_							:       Tuple       =				torch.load(lowerCamelCase_     ,					map_location='cpu'       )
     else:
          UpperCAmelCase_							:       int       =				torch.load(args.path_to_checkpoint     ,					map_location='cpu'       )

     UpperCAmelCase_							:       Union[str, Any]       =				input_state_dict.get('args'     ,					lowerCamelCase_       )

     # Read the config, or default to the model released by NVIDIA.
     if args.config_file == "":
          if ds_args is not None:
               if ds_args.bias_gelu_fusion:
                    UpperCAmelCase_							:       str       =				'gelu_fast'
               elif ds_args.openai_gelu:
                    UpperCAmelCase_							:       Tuple       =				'gelu_new'
               else:
                    UpperCAmelCase_							:       List[str]       =				'gelu'
          else:
               # in the very early days this used to be "gelu_new"
               UpperCAmelCase_							:       Optional[int]       =				'gelu_new'

          # Spell out all parameters in case the defaults change.
          UpperCAmelCase_							:       List[str]       =				GPTaConfig(
              vocab_size=50257     ,					n_positions=1024     ,					n_embd=1024     ,					n_layer=24     ,					n_head=16     ,					n_inner=4096     ,					activation_function=lowerCamelCase_     ,					resid_pdrop=0.1     ,					embd_pdrop=0.1     ,					attn_pdrop=0.1     ,					layer_norm_epsilon=1e-5     ,					initializer_range=0.02     ,					summary_type='cls_index'     ,					summary_use_proj=lowerCamelCase_     ,					summary_activation=lowerCamelCase_     ,					summary_proj_to_labels=lowerCamelCase_     ,					summary_first_dropout=0.1     ,					scale_attn_weights=lowerCamelCase_     ,					use_cache=lowerCamelCase_     ,					bos_token_id=50256     ,					eos_token_id=50256     ,					)
     else:
          UpperCAmelCase_							:       Any       =				GPTaConfig.from_json_file(args.config_file       )

     UpperCAmelCase_							:       Any       =				['GPT2LMHeadModel']

     # Convert.
     print('Converting'       )
     UpperCAmelCase_							:       Tuple       =				convert_megatron_checkpoint(lowerCamelCase_     ,					lowerCamelCase_     ,					lowerCamelCase_       )

     # Print the structure of converted state dict.
     if args.print_checkpoint_structure:
          recursive_print(lowerCamelCase_     ,					lowerCamelCase_       )

     # Add tokenizer class info to config
     # see https://github.com/huggingface/transformers/issues/13906)
     if ds_args is not None:
          UpperCAmelCase_							:       str       =				ds_args.tokenizer_type
          if tokenizer_type == "GPT2BPETokenizer":
               UpperCAmelCase_							:       Dict       =				'gpt2'
          elif tokenizer_type == "PretrainedFromHF":
               UpperCAmelCase_							:       Optional[int]       =				ds_args.tokenizer_name_or_path
          else:
               raise ValueError(F'''Unrecognized tokenizer_type {tokenizer_type}'''       )
     else:
          UpperCAmelCase_							:       Any       =				'gpt2'

     UpperCAmelCase_							:       Optional[Any]       =				AutoTokenizer.from_pretrained(lowerCamelCase_       )
     UpperCAmelCase_							:       Tuple       =				type(lowerCamelCase_       ).__name__
     UpperCAmelCase_							:       int       =				tokenizer_class

     # Store the config to file.
     print('Saving config'       )
     config.save_pretrained(lowerCamelCase_       )

     # Save tokenizer based on args
     print(F'''Adding {tokenizer_class} tokenizer files'''       )
     tokenizer.save_pretrained(lowerCamelCase_       )

     # Store the state_dict to file.
     UpperCAmelCase_							:       Optional[int]       =				os.path.join(lowerCamelCase_     ,					'pytorch_model.bin'       )
     print(F'''Saving checkpoint to "{output_checkpoint_file}"'''       )
     torch.save(lowerCamelCase_     ,					lowerCamelCase_       )


####################################################################################################

if __name__ == "__main__":
 main()

####################################################################################################







'''simple docstring'''


import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil






class 					__SCREAMING_SNAKE_CASE						(	lowerCamelCase_	):







  '''simple docstring'''






  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       Optional[Any]       =				pa.array(TypedSequence([1, 2, 3]						)						)
       self.assertEqual(arr.type				,		pa.intaa()						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       with self.assertRaises(snake_case_						):
            UpperCAmelCase_							:       Optional[int]       =				pa.array(TypedSequence([1, 2, 3]						)				,		type=pa.intaa()						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       with self.assertRaises(snake_case_						):
            UpperCAmelCase_							:       List[Any]       =				pa.array(TypedSequence([1, 2, 3]				,		try_type=Value('bool'						)				,		type=Value('int64'						)						)						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       Any       =				pa.array(TypedSequence([1, 2, 3]				,		type=Value('int32'						)						)						)
       self.assertEqual(arr.type				,		pa.intaa()						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       with self.assertRaises((TypeError, pa.lib.ArrowInvalid)						):
            UpperCAmelCase_							:       List[str]       =				pa.array(TypedSequence(['foo', 'bar']				,		type=Value('int64'						)						)						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       str       =				pa.array(TypedSequence([1, 2, 3]				,		try_type=Value('int32'						)						)						)
       self.assertEqual(arr.type				,		pa.intaa()						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       List[str]       =				pa.array(TypedSequence(['foo', 'bar']				,		try_type=Value('int64'						)						)						)
       self.assertEqual(arr.type				,		pa.string()						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       Tuple       =				pa.array(TypedSequence([[[1, 2, 3]]]				,		type=ArrayaD((1, 3)				,		'int64'						)						)						)
       self.assertEqual(arr.type				,		ArrayaDExtensionType((1, 3)				,		'int64'						)						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       with self.assertRaises((TypeError, pa.lib.ArrowInvalid)						):
            UpperCAmelCase_							:       Any       =				pa.array(TypedSequence(['foo', 'bar']				,		type=ArrayaD((1, 3)				,		'int64'						)						)						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       int       =				pa.array(TypedSequence([[[1, 2, 3]]]				,		try_type=ArrayaD((1, 3)				,		'int64'						)						)						)
       self.assertEqual(arr.type				,		ArrayaDExtensionType((1, 3)				,		'int64'						)						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       Union[str, Any]       =				pa.array(TypedSequence(['foo', 'bar']				,		try_type=ArrayaD((1, 3)				,		'int64'						)						)						)
       self.assertEqual(arr.type				,		pa.string()						)






  @require_pil
  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       import PIL.Image

       UpperCAmelCase_							:       Any       =				PIL.Image.fromarray(np.arange(1_0				,		dtype=np.uinta						).reshape(2				,		5						)						)
       with patch(
           'datasets.arrow_writer.cast_to_python_objects'				,		side_effect=snake_case_						) as mock_cast_to_python_objects:
            UpperCAmelCase_							:       int       =				pa.array(TypedSequence([{'path': None, 'bytes': b'image_bytes'}, pil_image]				,		type=Image()						)						)
            UpperCAmelCase_       ,      UpperCAmelCase_							:       Any       =				mock_cast_to_python_objects.call_args_list[-1]
            self.assertIn('optimize_list_casting'				,		snake_case_						)
            self.assertFalse(kwargs['optimize_list_casting']						)







def  _lowerCamelCase   (				lowerCamelCase_ :							Any     ,					lowerCamelCase_ :							int       ):



     """simple docstring"""





     UpperCAmelCase_							:       str       =				pa.BufferReader(lowerCamelCase_       ) if isinstance(lowerCamelCase_     ,					pa.Buffer       ) else pa.memory_map(lowerCamelCase_       )
     UpperCAmelCase_							:       Union[str, Any]       =				pa.ipc.open_stream(lowerCamelCase_       )
     UpperCAmelCase_							:       pa.Table       =				f.read_all()
     assert len(pa_table.to_batches()       ) == expected_num_chunks
     assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
     del pa_table


@pytest.mark.parametrize('writer_batch_size'     ,					[None, 1, 10]       )
@pytest.mark.parametrize(
    'fields'     ,					[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]       )
def  _lowerCamelCase   (				lowerCamelCase_ :							Optional[Any]     ,					lowerCamelCase_ :							int       ):



     """simple docstring"""





     UpperCAmelCase_							:       List[str]       =				pa.BufferOutputStream()
     UpperCAmelCase_							:       Dict       =				pa.schema(lowerCamelCase_       ) if fields else None
     with ArrowWriter(stream=lowerCamelCase_     ,					schema=lowerCamelCase_     ,					writer_batch_size=lowerCamelCase_       ) as writer:
          writer.write({'col_1': 'foo', 'col_2': 1}       )
          writer.write({'col_1': 'bar', 'col_2': 2}       )
          UpperCAmelCase_       ,      UpperCAmelCase_							:       Dict       =				writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     if not fields:
          UpperCAmelCase_							:       Optional[int]       =				{'col_1': pa.string(), 'col_2': pa.intaa()}
     assert writer._schema == pa.schema(lowerCamelCase_     ,					metadata=writer._schema.metadata       )
     _check_output(output.getvalue()     ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1       )


def  _lowerCamelCase   (				):



     """simple docstring"""





     UpperCAmelCase_							:       Tuple       =				pa.BufferOutputStream()
     UpperCAmelCase_							:       Optional[int]       =				Features({'labels': ClassLabel(names=['neg', 'pos']       )}       )
     with ArrowWriter(stream=lowerCamelCase_     ,					features=lowerCamelCase_       ) as writer:
          writer.write({'labels': 0}       )
          writer.write({'labels': 1}       )
          UpperCAmelCase_       ,      UpperCAmelCase_							:       List[Any]       =				writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     assert writer._schema == features.arrow_schema
     assert writer._schema.metadata == features.arrow_schema.metadata
     UpperCAmelCase_							:       Any       =				pa.BufferReader(output.getvalue()       )
     UpperCAmelCase_							:       List[str]       =				pa.ipc.open_stream(lowerCamelCase_       )
     UpperCAmelCase_							:       pa.Table       =				f.read_all()
     UpperCAmelCase_							:       Optional[Any]       =				pa_table.schema
     assert pa_table.num_rows == 2
     assert schema == features.arrow_schema
     assert schema.metadata == features.arrow_schema.metadata
     assert features == Features.from_arrow_schema(lowerCamelCase_       )


@pytest.mark.parametrize('writer_batch_size'     ,					[None, 1, 10]       )
def  _lowerCamelCase   (				lowerCamelCase_ :							List[Any]       ):



     """simple docstring"""





     UpperCAmelCase_							:       Union[str, Any]       =				pa.BufferOutputStream()
     with ArrowWriter(
         stream=lowerCamelCase_     ,					writer_batch_size=lowerCamelCase_     ,					hash_salt='split_name'     ,					check_duplicates=lowerCamelCase_     ,					) as writer:
          with pytest.raises(lowerCamelCase_       ):
               writer.write({'col_1': 'foo', 'col_2': 1}     ,					key=[1, 2]       )
               UpperCAmelCase_       ,      UpperCAmelCase_							:       Any       =				writer.finalize()


@pytest.mark.parametrize('writer_batch_size'     ,					[None, 2, 10]       )
def  _lowerCamelCase   (				lowerCamelCase_ :							Union[str, Any]       ):



     """simple docstring"""





     UpperCAmelCase_							:       int       =				pa.BufferOutputStream()
     with ArrowWriter(
         stream=lowerCamelCase_     ,					writer_batch_size=lowerCamelCase_     ,					hash_salt='split_name'     ,					check_duplicates=lowerCamelCase_     ,					) as writer:
          with pytest.raises(lowerCamelCase_       ):
               writer.write({'col_1': 'foo', 'col_2': 1}     ,					key=10       )
               writer.write({'col_1': 'bar', 'col_2': 2}     ,					key=10       )
               UpperCAmelCase_       ,      UpperCAmelCase_							:       Optional[Any]       =				writer.finalize()


@pytest.mark.parametrize('writer_batch_size'     ,					[None, 2, 10]       )
def  _lowerCamelCase   (				lowerCamelCase_ :							Optional[int]       ):



     """simple docstring"""





     UpperCAmelCase_							:       str       =				pa.BufferOutputStream()
     with ArrowWriter(
         stream=lowerCamelCase_     ,					writer_batch_size=lowerCamelCase_     ,					hash_salt='split_name'     ,					check_duplicates=lowerCamelCase_     ,					) as writer:
          writer.write({'col_1': 'foo', 'col_2': 1}     ,					key=1       )
          writer.write({'col_1': 'bar', 'col_2': 2}     ,					key=2       )
          UpperCAmelCase_       ,      UpperCAmelCase_							:       Dict       =				writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     _check_output(output.getvalue()     ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1       )


@pytest.mark.parametrize('writer_batch_size'     ,					[None, 1, 10]       )
@pytest.mark.parametrize(
    'fields'     ,					[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]       )
def  _lowerCamelCase   (				lowerCamelCase_ :							Dict     ,					lowerCamelCase_ :							str       ):



     """simple docstring"""





     UpperCAmelCase_							:       List[Any]       =				pa.BufferOutputStream()
     UpperCAmelCase_							:       Optional[Any]       =				pa.schema(lowerCamelCase_       ) if fields else None
     with ArrowWriter(stream=lowerCamelCase_     ,					schema=lowerCamelCase_     ,					writer_batch_size=lowerCamelCase_       ) as writer:
          writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}       )
          writer.write_batch({'col_1': [], 'col_2': []}       )
          UpperCAmelCase_       ,      UpperCAmelCase_							:       str       =				writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     if not fields:
          UpperCAmelCase_							:       int       =				{'col_1': pa.string(), 'col_2': pa.intaa()}
     assert writer._schema == pa.schema(lowerCamelCase_     ,					metadata=writer._schema.metadata       )
     _check_output(output.getvalue()     ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1       )


@pytest.mark.parametrize('writer_batch_size'     ,					[None, 1, 10]       )
@pytest.mark.parametrize(
    'fields'     ,					[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]       )
def  _lowerCamelCase   (				lowerCamelCase_ :							int     ,					lowerCamelCase_ :							Union[str, Any]       ):



     """simple docstring"""





     UpperCAmelCase_							:       List[str]       =				pa.BufferOutputStream()
     UpperCAmelCase_							:       Optional[int]       =				pa.schema(lowerCamelCase_       ) if fields else None
     with ArrowWriter(stream=lowerCamelCase_     ,					schema=lowerCamelCase_     ,					writer_batch_size=lowerCamelCase_       ) as writer:
          writer.write_table(pa.Table.from_pydict({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}       )       )
          UpperCAmelCase_       ,      UpperCAmelCase_							:       Union[str, Any]       =				writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     if not fields:
          UpperCAmelCase_							:       Any       =				{'col_1': pa.string(), 'col_2': pa.intaa()}
     assert writer._schema == pa.schema(lowerCamelCase_     ,					metadata=writer._schema.metadata       )
     _check_output(output.getvalue()     ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1       )


@pytest.mark.parametrize('writer_batch_size'     ,					[None, 1, 10]       )
@pytest.mark.parametrize(
    'fields'     ,					[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]       )
def  _lowerCamelCase   (				lowerCamelCase_ :							Dict     ,					lowerCamelCase_ :							List[str]       ):



     """simple docstring"""





     UpperCAmelCase_							:       Tuple       =				pa.BufferOutputStream()
     UpperCAmelCase_							:       Tuple       =				pa.schema(lowerCamelCase_       ) if fields else None
     with ArrowWriter(stream=lowerCamelCase_     ,					schema=lowerCamelCase_     ,					writer_batch_size=lowerCamelCase_       ) as writer:
          writer.write_row(pa.Table.from_pydict({'col_1': ['foo'], 'col_2': [1]}       )       )
          writer.write_row(pa.Table.from_pydict({'col_1': ['bar'], 'col_2': [2]}       )       )
          UpperCAmelCase_       ,      UpperCAmelCase_							:       Optional[int]       =				writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     if not fields:
          UpperCAmelCase_							:       Union[str, Any]       =				{'col_1': pa.string(), 'col_2': pa.intaa()}
     assert writer._schema == pa.schema(lowerCamelCase_     ,					metadata=writer._schema.metadata       )
     _check_output(output.getvalue()     ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1       )


def  _lowerCamelCase   (				):



     """simple docstring"""





     with tempfile.TemporaryDirectory() as tmp_dir:
          UpperCAmelCase_							:       Union[str, Any]       =				{'col_1': pa.string(), 'col_2': pa.intaa()}
          UpperCAmelCase_							:       Tuple       =				os.path.join(lowerCamelCase_     ,					'test.arrow'       )
          with ArrowWriter(path=lowerCamelCase_     ,					schema=pa.schema(lowerCamelCase_       )       ) as writer:
               writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}       )
               UpperCAmelCase_       ,      UpperCAmelCase_							:       Union[str, Any]       =				writer.finalize()
          assert num_examples == 2
          assert num_bytes > 0
          assert writer._schema == pa.schema(lowerCamelCase_     ,					metadata=writer._schema.metadata       )
          _check_output(lowerCamelCase_     ,					1       )


def  _lowerCamelCase   (				lowerCamelCase_ :							Optional[Any]       ):



     """simple docstring"""





     if pa.types.is_list(lowerCamelCase_       ):
          return get_base_dtype(arr_type.value_type       )
     else:
          return arr_type


def  _lowerCamelCase   (				lowerCamelCase_ :							Optional[int]     ,					lowerCamelCase_ :							Optional[Any]       ):



     """simple docstring"""





     if isinstance(lst[0]     ,					lowerCamelCase_       ):
          change_first_primitive_element_in_list(lst[0]     ,					lowerCamelCase_       )
     else:
          UpperCAmelCase_							:       str       =				value


@pytest.mark.parametrize('optimized_int_type, expected_dtype'     ,					[(None, pa.intaa()), (Value('int32'       ), pa.intaa())]       )
@pytest.mark.parametrize('sequence'     ,					[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]       )
def  _lowerCamelCase   (				lowerCamelCase_ :							int     ,					lowerCamelCase_ :							Dict     ,					lowerCamelCase_ :							int       ):



     """simple docstring"""





     UpperCAmelCase_							:       List[Any]       =				pa.array(TypedSequence(lowerCamelCase_     ,					optimized_int_type=lowerCamelCase_       )       )
     assert get_base_dtype(arr.type       ) == expected_dtype


@pytest.mark.parametrize(
    'col, expected_dtype'     ,					[
        ('attention_mask', pa.inta()),
        ('special_tokens_mask', pa.inta()),
        ('token_type_ids', pa.inta()),
        ('input_ids', pa.intaa()),
        ('other', pa.intaa()),
    ]     ,					)
@pytest.mark.parametrize('sequence'     ,					[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]       )
def  _lowerCamelCase   (				lowerCamelCase_ :							Optional[Any]     ,					lowerCamelCase_ :							Tuple     ,					lowerCamelCase_ :							List[Any]       ):



     """simple docstring"""





     UpperCAmelCase_							:       Tuple       =				pa.array(OptimizedTypedSequence(lowerCamelCase_     ,					col=lowerCamelCase_       )       )
     assert get_base_dtype(arr.type       ) == expected_dtype

     # not in range
     if col != "other":
          # avoids errors due to in-place modifications
          UpperCAmelCase_							:       int       =				copy.deepcopy(lowerCamelCase_       )
          UpperCAmelCase_							:       str       =				np.iinfo(expected_dtype.to_pandas_dtype()       ).max + 1
          change_first_primitive_element_in_list(lowerCamelCase_     ,					lowerCamelCase_       )
          UpperCAmelCase_							:       Dict       =				pa.array(OptimizedTypedSequence(lowerCamelCase_     ,					col=lowerCamelCase_       )       )
          assert get_base_dtype(arr.type       ) == pa.intaa()


@pytest.mark.parametrize('raise_exception'     ,					[False, True]       )
def  _lowerCamelCase   (				lowerCamelCase_ :							List[str]     ,					lowerCamelCase_ :							List[Any]       ):



     """simple docstring"""





     UpperCAmelCase_							:       int       =				str(tmp_path / 'dataset-train.arrow'       )
     try:
          with ArrowWriter(path=lowerCamelCase_       ) as writer:
               if raise_exception:
                    raise pa.lib.ArrowInvalid()
               else:
                    writer.stream.close()
     except pa.lib.ArrowInvalid:
          pass
     finally:
          assert writer.stream.closed


def  _lowerCamelCase   (				lowerCamelCase_ :							Dict       ):



     """simple docstring"""





     UpperCAmelCase_							:       Dict       =				'mock://dataset-train.arrow'
     with ArrowWriter(path=lowerCamelCase_     ,					storage_options=mockfs.storage_options       ) as writer:
          assert isinstance(writer._fs     ,					type(lowerCamelCase_       )       )
          assert writer._fs.storage_options == mockfs.storage_options
          writer.write({'col_1': 'foo', 'col_2': 1}       )
          writer.write({'col_1': 'bar', 'col_2': 2}       )
          UpperCAmelCase_       ,      UpperCAmelCase_							:       Tuple       =				writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     assert mockfs.exists(lowerCamelCase_       )


def  _lowerCamelCase   (				):



     """simple docstring"""





     UpperCAmelCase_							:       Optional[Any]       =				pa.BufferOutputStream()
     with ParquetWriter(stream=lowerCamelCase_       ) as writer:
          writer.write({'col_1': 'foo', 'col_2': 1}       )
          writer.write({'col_1': 'bar', 'col_2': 2}       )
          UpperCAmelCase_       ,      UpperCAmelCase_							:       Optional[int]       =				writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     UpperCAmelCase_							:       List[str]       =				pa.BufferReader(output.getvalue()       )
     UpperCAmelCase_							:       pa.Table       =				pq.read_table(lowerCamelCase_       )
     assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}


@require_pil
@pytest.mark.parametrize('embed_local_files'     ,					[False, True]       )
def  _lowerCamelCase   (				lowerCamelCase_ :							List[Any]     ,					lowerCamelCase_ :							Union[str, Any]       ):



     """simple docstring"""





     import PIL.Image

     UpperCAmelCase_							:       str       =				str(tmp_path / 'test_image_rgb.jpg'       )
     PIL.Image.fromarray(np.zeros((5, 5)     ,					dtype=np.uinta       )       ).save(lowerCamelCase_     ,					format='png'       )
     UpperCAmelCase_							:       int       =				pa.BufferOutputStream()
     with ParquetWriter(
         stream=lowerCamelCase_     ,					features=Features({'image': Image()}       )     ,					embed_local_files=lowerCamelCase_       ) as writer:
          writer.write({'image': image_path}       )
          writer.finalize()
     UpperCAmelCase_							:       Optional[int]       =				pa.BufferReader(output.getvalue()       )
     UpperCAmelCase_							:       pa.Table       =				pq.read_table(lowerCamelCase_       )
     UpperCAmelCase_							:       List[str]       =				pa_table.to_pydict()
     if embed_local_files:
          assert isinstance(out['image'][0]['path']     ,					lowerCamelCase_       )
          with open(lowerCamelCase_     ,					'rb'       ) as f:
               assert out["image"][0]["bytes"] == f.read()
     else:
          assert out["image"][0]["path"] == image_path
          assert out["image"][0]["bytes"] is None





def  _lowerCamelCase   (				):



     """simple docstring"""





     UpperCAmelCase_							:       List[str]       =				pa.schema([pa.field('col_1'     ,					pa.string()     ,					nullable=lowerCamelCase_       )]       )
     UpperCAmelCase_							:       Dict       =				pa.BufferOutputStream()
     with ArrowWriter(stream=lowerCamelCase_       ) as writer:
          writer._build_writer(inferred_schema=lowerCamelCase_       )
          assert writer._schema == pa.schema([pa.field('col_1'     ,					pa.string()       )]       )






