
import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig
def 							SCREAMING_SNAKE_CASE_	(					UpperCAmelCase_	:     Dict			,							UpperCAmelCase_	:     Any			,							UpperCAmelCase_	:     Tuple=0				)			->						Any:
       # Format the message.
       if name is None:
              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=None
       else:
              SCREAMING_SNAKE_CASE_							:				List[Any]		='.' * max(0			,							spaces - 2				) + '# {:' + str(5_0 - spaces				) + 's}'
              SCREAMING_SNAKE_CASE_							:				Dict		=fmt.format(__snake_case				)

       # Print and recurse (if needed).
       if isinstance(__snake_case			,							__snake_case				):
              if msg is not None:
                     print(__snake_case				)
              for k in val.keys():
                     recursive_print(__snake_case			,							val[k]			,							spaces + 2				)
       elif isinstance(__snake_case			,							torch.Tensor				):
              print(__snake_case			,							''':'''			,							val.size()				)
       else:
              print(__snake_case			,							''':'''			,							__snake_case				)
def 							SCREAMING_SNAKE_CASE_	(					UpperCAmelCase_	:     Union[str, Any]			,							UpperCAmelCase_	:     List[str]			,							UpperCAmelCase_	:     Optional[Any]			,							UpperCAmelCase_	:     List[str]			,							UpperCAmelCase_	:     List[str]				)			->						Union[str, Any]:
       # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
       # for compatibility with later versions of NVIDIA Megatron-LM.
       # The inverse operation is performed inside Megatron-LM to read checkpoints:
       # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
       # If param is the weight tensor of the self-attention block, the returned tensor
       # will have to be transposed one more time to be read by HuggingFace GPT2.
       SCREAMING_SNAKE_CASE_							:				Optional[Any]		=param.size()
       if checkpoint_version == 1.0:
              # version 1.0 stores [num_heads * hidden_size * num_splits, :]
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=(num_heads, hidden_size, num_splits) + input_shape[1:]
              SCREAMING_SNAKE_CASE_							:				List[str]		=param.view(*__snake_case				)
              SCREAMING_SNAKE_CASE_							:				Tuple		=param.transpose(0			,							2				)
              SCREAMING_SNAKE_CASE_							:				Dict		=param.transpose(1			,							2				).contiguous()
       elif checkpoint_version >= 2.0:
              # other versions store [num_heads * num_splits * hidden_size, :]
              SCREAMING_SNAKE_CASE_							:				Dict		=(num_heads, num_splits, hidden_size) + input_shape[1:]
              SCREAMING_SNAKE_CASE_							:				str		=param.view(*__snake_case				)
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=param.transpose(0			,							1				).contiguous()
       SCREAMING_SNAKE_CASE_							:				Dict		=param.view(*__snake_case				)
       return param
def 							SCREAMING_SNAKE_CASE_	(					UpperCAmelCase_	:     Optional[int]			,							UpperCAmelCase_	:     str			,							UpperCAmelCase_	:     Dict				)			->						Union[str, Any]:
       # The converted output model.
       SCREAMING_SNAKE_CASE_							:				str		={}

       # old versions did not store training args
       SCREAMING_SNAKE_CASE_							:				List[str]		=input_state_dict.get('''args'''			,							__snake_case				)
       if ds_args is not None:
              # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
              # from pprint import pprint
              # pprint(vars(ds_args))

              SCREAMING_SNAKE_CASE_							:				Tuple		=ds_args.padded_vocab_size
              SCREAMING_SNAKE_CASE_							:				str		=ds_args.max_position_embeddings
              SCREAMING_SNAKE_CASE_							:				str		=ds_args.hidden_size
              SCREAMING_SNAKE_CASE_							:				Any		=ds_args.num_layers
              SCREAMING_SNAKE_CASE_							:				List[Any]		=ds_args.num_attention_heads
              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=ds_args.ffn_hidden_size
              # pprint(config)

       # The number of heads.
       SCREAMING_SNAKE_CASE_							:				List[Any]		=config.n_head
       # The hidden_size per head.
       SCREAMING_SNAKE_CASE_							:				List[str]		=config.n_embd // config.n_head
       # Megatron-LM checkpoint version
       if "checkpoint_version" in input_state_dict.keys():
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=input_state_dict['checkpoint_version']
       else:
              SCREAMING_SNAKE_CASE_							:				List[str]		=0.0

       # The model.
       SCREAMING_SNAKE_CASE_							:				Any		=input_state_dict['model']
       # The language model.
       SCREAMING_SNAKE_CASE_							:				Tuple		=model['language_model']
       # The embeddings.
       SCREAMING_SNAKE_CASE_							:				Any		=lm['embedding']

       # The word embeddings.
       SCREAMING_SNAKE_CASE_							:				int		=embeddings['word_embeddings']['weight']
       # Truncate the embedding table to vocab_size rows.
       SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=word_embeddings[: config.vocab_size, :]
       SCREAMING_SNAKE_CASE_							:				Any		=word_embeddings

       # The position embeddings.
       SCREAMING_SNAKE_CASE_							:				List[Any]		=embeddings['position_embeddings']['weight']
       # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
       SCREAMING_SNAKE_CASE_							:				Dict		=pos_embeddings.size(0				)
       if n_positions != config.n_positions:
              raise ValueError(
                  f'pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'				)
       # Store the position embeddings.
       SCREAMING_SNAKE_CASE_							:				Any		=pos_embeddings

       # The transformer.
       SCREAMING_SNAKE_CASE_							:				Any		=lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

       # The regex to extract layer names.
       SCREAMING_SNAKE_CASE_							:				Optional[Any]		=re.compile(R'''layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'''				)

       # The simple map of names for "automated" rules.
       SCREAMING_SNAKE_CASE_							:				int		={
           'attention.dense': '.attn.c_proj.',
           'self_attention.dense': '.attn.c_proj.',
           'mlp.dense_h_to_4h': '.mlp.c_fc.',
           'mlp.dense_4h_to_h': '.mlp.c_proj.',
       }

       # Extract the layers.
       for key, val in transformer.items():
              # Match the name.
              SCREAMING_SNAKE_CASE_							:				str		=layer_re.match(__snake_case				)

              # Stop if that's not a layer
              if m is None:
                     break

              # The index of the layer.
              SCREAMING_SNAKE_CASE_							:				Any		=int(m.group(1				)				)
              # The name of the operation.
              SCREAMING_SNAKE_CASE_							:				List[Any]		=m.group(2				)
              # Is it a weight or a bias?
              SCREAMING_SNAKE_CASE_							:				str		=m.group(3				)

              # The name of the layer.
              SCREAMING_SNAKE_CASE_							:				Dict		=f'transformer.h.{layer_idx}'

              # For layernorm(s), simply store the layer norm.
              if op_name.endswith('''layernorm'''				):
                     SCREAMING_SNAKE_CASE_							:				str		='ln_1' if op_name.startswith('''input'''				) else 'ln_2'
                     SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=val

              # Transpose the QKV matrix.
              elif (
                  op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
              ) and weight_or_bias == "weight":
                     # Insert a tensor of 1x1xDxD bias.
                     SCREAMING_SNAKE_CASE_							:				Dict		=torch.tril(torch.ones((n_positions, n_positions)			,							dtype=torch.floataa				)				).view(
                         1			,							1			,							__snake_case			,							__snake_case				)
                     SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=causal_mask

                     # Insert a "dummy" tensor for masked_bias.
                     SCREAMING_SNAKE_CASE_							:				Any		=torch.tensor(-1E4			,							dtype=torch.floataa				)
                     SCREAMING_SNAKE_CASE_							:				List[Any]		=masked_bias

                     SCREAMING_SNAKE_CASE_							:				List[Any]		=fix_query_key_value_ordering(__snake_case			,							__snake_case			,							3			,							__snake_case			,							__snake_case				)
                     # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
                     SCREAMING_SNAKE_CASE_							:				List[Any]		=out_val.transpose(0			,							1				).contiguous()
                     # Store.
                     SCREAMING_SNAKE_CASE_							:				List[Any]		=out_val

              # Transpose the bias.
              elif (
                  op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
              ) and weight_or_bias == "bias":
                     SCREAMING_SNAKE_CASE_							:				Any		=fix_query_key_value_ordering(__snake_case			,							__snake_case			,							3			,							__snake_case			,							__snake_case				)
                     # Store. No change of shape.
                     SCREAMING_SNAKE_CASE_							:				Optional[Any]		=out_val

              # Transpose the weights.
              elif weight_or_bias == "weight":
                     SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=megatron_to_transformers[op_name]
                     SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=val.transpose(0			,							1				)

              # Copy the bias.
              elif weight_or_bias == "bias":
                     SCREAMING_SNAKE_CASE_							:				Optional[Any]		=megatron_to_transformers[op_name]
                     SCREAMING_SNAKE_CASE_							:				Tuple		=val

    # DEBUG.
       assert config.n_layer == layer_idx + 1

       # The final layernorm.
       SCREAMING_SNAKE_CASE_							:				List[Any]		=transformer['final_layernorm.weight']
       SCREAMING_SNAKE_CASE_							:				int		=transformer['final_layernorm.bias']

       # For LM head, transformers' wants the matrix to weight embeddings.
       SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=word_embeddings

       # It should be done!
       return output_state_dict
def 							SCREAMING_SNAKE_CASE_	(					)			->						Optional[Any]:
       # Create the argument parser.
       SCREAMING_SNAKE_CASE_							:				List[Any]		=argparse.ArgumentParser()
       parser.add_argument('''--print-checkpoint-structure'''			,							action='''store_true'''				)
       parser.add_argument(
           '''path_to_checkpoint'''			,							type=__snake_case			,							help='''Path to the checkpoint file (.zip archive or direct .pt file)'''			,							)
       parser.add_argument(
           '''--config_file'''			,							default=''''''			,							type=__snake_case			,							help='''An optional config json file describing the pre-trained model.'''			,							)
       SCREAMING_SNAKE_CASE_							:				Optional[Any]		=parser.parse_args()

       # Extract the basename.
       SCREAMING_SNAKE_CASE_							:				List[Any]		=os.path.dirname(args.path_to_checkpoint				)

       # Load the model.
       # the .zip is very optional, let's keep it for backward compatibility
       print(f'Extracting PyTorch state dictionary from {args.path_to_checkpoint}'				)
       if args.path_to_checkpoint.endswith('''.zip'''				):
              with zipfile.ZipFile(args.path_to_checkpoint			,							'''r'''				) as checkpoint:
                     with checkpoint.open('''release/mp_rank_00/model_optim_rng.pt'''				) as pytorch_dict:
                            SCREAMING_SNAKE_CASE_							:				Optional[Any]		=torch.load(__snake_case			,							map_location='''cpu'''				)
       else:
              SCREAMING_SNAKE_CASE_							:				List[Any]		=torch.load(args.path_to_checkpoint			,							map_location='''cpu'''				)

       SCREAMING_SNAKE_CASE_							:				Dict		=input_state_dict.get('''args'''			,							__snake_case				)

       # Read the config, or default to the model released by NVIDIA.
       if args.config_file == "":
              if ds_args is not None:
                     if ds_args.bias_gelu_fusion:
                            SCREAMING_SNAKE_CASE_							:				int		='gelu_fast'
                     elif ds_args.openai_gelu:
                            SCREAMING_SNAKE_CASE_							:				Dict		='gelu_new'
                     else:
                            SCREAMING_SNAKE_CASE_							:				int		='gelu'
              else:
                     # in the very early days this used to be "gelu_new"
                     SCREAMING_SNAKE_CASE_							:				Optional[Any]		='gelu_new'

              # Spell out all parameters in case the defaults change.
              SCREAMING_SNAKE_CASE_							:				str		=GPTaConfig(
                  vocab_size=5_0_2_5_7			,							n_positions=1_0_2_4			,							n_embd=1_0_2_4			,							n_layer=2_4			,							n_head=1_6			,							n_inner=4_0_9_6			,							activation_function=__snake_case			,							resid_pdrop=0.1			,							embd_pdrop=0.1			,							attn_pdrop=0.1			,							layer_norm_epsilon=1E-5			,							initializer_range=0.02			,							summary_type='''cls_index'''			,							summary_use_proj=__snake_case			,							summary_activation=__snake_case			,							summary_proj_to_labels=__snake_case			,							summary_first_dropout=0.1			,							scale_attn_weights=__snake_case			,							use_cache=__snake_case			,							bos_token_id=5_0_2_5_6			,							eos_token_id=5_0_2_5_6			,							)
       else:
              SCREAMING_SNAKE_CASE_							:				int		=GPTaConfig.from_json_file(args.config_file				)

       SCREAMING_SNAKE_CASE_							:				Optional[Any]		=['GPT2LMHeadModel']

       # Convert.
       print('''Converting'''				)
       SCREAMING_SNAKE_CASE_							:				str		=convert_megatron_checkpoint(__snake_case			,							__snake_case			,							__snake_case				)

       # Print the structure of converted state dict.
       if args.print_checkpoint_structure:
              recursive_print(__snake_case			,							__snake_case				)

       # Add tokenizer class info to config
       # see https://github.com/huggingface/transformers/issues/13906)
       if ds_args is not None:
              SCREAMING_SNAKE_CASE_							:				List[Any]		=ds_args.tokenizer_type
              if tokenizer_type == "GPT2BPETokenizer":
                     SCREAMING_SNAKE_CASE_							:				Tuple		='gpt2'
              elif tokenizer_type == "PretrainedFromHF":
                     SCREAMING_SNAKE_CASE_							:				Optional[int]		=ds_args.tokenizer_name_or_path
              else:
                     raise ValueError(f'Unrecognized tokenizer_type {tokenizer_type}'				)
       else:
              SCREAMING_SNAKE_CASE_							:				int		='gpt2'

       SCREAMING_SNAKE_CASE_							:				Optional[int]		=AutoTokenizer.from_pretrained(__snake_case				)
       SCREAMING_SNAKE_CASE_							:				List[str]		=type(__snake_case				).__name__
       SCREAMING_SNAKE_CASE_							:				int		=tokenizer_class

       # Store the config to file.
       print('''Saving config'''				)
       config.save_pretrained(__snake_case				)

       # Save tokenizer based on args
       print(f'Adding {tokenizer_class} tokenizer files'				)
       tokenizer.save_pretrained(__snake_case				)

       # Store the state_dict to file.
       SCREAMING_SNAKE_CASE_							:				Dict		=os.path.join(__snake_case			,							'''pytorch_model.bin'''				)
       print(f'Saving checkpoint to "{output_checkpoint_file}"'				)
       torch.save(__snake_case			,							__snake_case				)


####################################################################################################

if __name__ == "__main__":
 main()

####################################################################################################







'''simple docstring'''




from ...configuration_utils import PretrainedConfig


lowercase__       : Any         =       {
    '''google/tapas-base-finetuned-sqa''': (
        '''https://huggingface.co/google/tapas-base-finetuned-sqa/resolve/main/config.json'''
    ),
    '''google/tapas-base-finetuned-wtq''': (
        '''https://huggingface.co/google/tapas-base-finetuned-wtq/resolve/main/config.json'''
    ),
    '''google/tapas-base-finetuned-wikisql-supervised''': (
        '''https://huggingface.co/google/tapas-base-finetuned-wikisql-supervised/resolve/main/config.json'''
    ),
    '''google/tapas-base-finetuned-tabfact''': (
        '''https://huggingface.co/google/tapas-base-finetuned-tabfact/resolve/main/config.json'''
    ),
}


class 					SCREAMING_SNAKE_CASE       (a__				):
     lowerCAmelCase						     =			'''tapas'''


     def __init__(		self		,      _UpperCAmelCase=3_0522		,      _UpperCAmelCase=768		,      _UpperCAmelCase=12		,      _UpperCAmelCase=12		,      _UpperCAmelCase=3072		,      _UpperCAmelCase="gelu"		,      _UpperCAmelCase=0.1		,      _UpperCAmelCase=0.1		,      _UpperCAmelCase=1024		,      _UpperCAmelCase=[3, 256, 256, 2, 256, 256, 10]		,      _UpperCAmelCase=0.02		,      _UpperCAmelCase=1e-1_2		,      _UpperCAmelCase=0		,      _UpperCAmelCase=10.0		,      _UpperCAmelCase=0		,      _UpperCAmelCase=1.0		,      _UpperCAmelCase=None		,      _UpperCAmelCase=1.0		,      _UpperCAmelCase=False		,      _UpperCAmelCase=None		,      _UpperCAmelCase=1.0		,      _UpperCAmelCase=1.0		,      _UpperCAmelCase=False		,      _UpperCAmelCase=False		,      _UpperCAmelCase="ratio"		,      _UpperCAmelCase=None		,      _UpperCAmelCase=None		,      _UpperCAmelCase=64		,      _UpperCAmelCase=32		,      _UpperCAmelCase=False		,      _UpperCAmelCase=True		,      _UpperCAmelCase=False		,      _UpperCAmelCase=False		,      _UpperCAmelCase=True		,      _UpperCAmelCase=False		,      _UpperCAmelCase=None		,      _UpperCAmelCase=None		,      **_UpperCAmelCase		,      ):

          '''simple docstring'''






          super().__init__(pad_token_id=_UpperCAmelCase		,      **_UpperCAmelCase)

          # BERT hyperparameters (with updated max_position_embeddings and type_vocab_sizes)
          __A		:					Dict						=			vocab_size
          __A		:					Tuple						=			hidden_size
          __A		:					Any						=			num_hidden_layers
          __A		:					int						=			num_attention_heads
          __A		:					Tuple						=			hidden_act
          __A		:					Tuple						=			intermediate_size
          __A		:					List[Any]						=			hidden_dropout_prob
          __A		:					int						=			attention_probs_dropout_prob
          __A		:					List[str]						=			max_position_embeddings
          __A		:					Optional[int]						=			type_vocab_sizes
          __A		:					str						=			initializer_range
          __A		:					List[str]						=			layer_norm_eps

          # Fine-tuning task hyperparameters
          __A		:					List[str]						=			positive_label_weight
          __A		:					List[Any]						=			num_aggregation_labels
          __A		:					Optional[Any]						=			aggregation_loss_weight
          __A		:					Tuple						=			use_answer_as_supervision
          __A		:					List[str]						=			answer_loss_importance
          __A		:					Any						=			use_normalized_answer_loss
          __A		:					Any						=			huber_loss_delta
          __A		:					Union[str, Any]						=			temperature
          __A		:					Tuple						=			aggregation_temperature
          __A		:					Optional[Any]						=			use_gumbel_for_cells
          __A		:					List[str]						=			use_gumbel_for_aggregation
          __A		:					Tuple						=			average_approximation_function
          __A		:					List[str]						=			cell_selection_preference
          __A		:					Dict						=			answer_loss_cutoff
          __A		:					Union[str, Any]						=			max_num_rows
          __A		:					Optional[Any]						=			max_num_columns
          __A		:					int						=			average_logits_per_cell
          __A		:					Optional[Any]						=			select_one_column
          __A		:					int						=			allow_empty_column_selection
          __A		:					List[Any]						=			init_cell_selection_weights_to_zero
          __A		:					int						=			reset_position_index_per_cell
          __A		:					Union[str, Any]						=			disable_per_token_loss

          # Aggregation hyperparameters
          __A		:					Optional[Any]						=			aggregation_labels
          __A		:					List[str]						=			no_aggregation_label_index

          if isinstance(self.aggregation_labels		,      _UpperCAmelCase):
               __A		:					Optional[Any]						=			{int(_UpperCAmelCase): v for k, v in aggregation_labels.items()}