


'''simple docstring'''






import argparse
import os
import re

import tensorflow as tf
import torch

from transformers import BertConfig, BertModel
from transformers.utils import logging


logging.set_verbosity_info()
__UpperCAmelCase       							=				logging.get_logger(__name__)


def    _snake_case  (	A					,       A					,       A			)		->       Union[str, Any]:
							lowerCAmelCase__			  =      os.path.abspath(A			)
							logger.info(F"""Converting TensorFlow checkpoint from {tf_path}"""			)
							# Load weights from TF model
							lowerCAmelCase__			  =      tf.train.list_variables(A			)
							lowerCAmelCase__			  =      []
							lowerCAmelCase__			  =      []
							lowerCAmelCase__			  =      []
							for full_name, shape in init_vars:
														# logger.info(f"Loading TF weight {name} with shape {shape}")
														lowerCAmelCase__			  =      full_name.split('''/'''			)
														if full_name == "_CHECKPOINTABLE_OBJECT_GRAPH" or name[0] in ["global_step", "save_counter"]:
																					logger.info(F"""Skipping non-model layer {full_name}"""			)
																					continue
														if "optimizer" in full_name:
																					logger.info(F"""Skipping optimization layer {full_name}"""			)
																					continue
														if name[0] == "model":
																					# ignore initial 'model'
																					lowerCAmelCase__			  =      name[1:]
														# figure out how many levels deep the name is
														lowerCAmelCase__			  =      0
														for _name in name:
																					if _name.startswith('''layer_with_weights'''			):
																												depth += 1
																					else:
																												break
														layer_depth.append(A			)
														# read data
														lowerCAmelCase__			  =      tf.train.load_variable(A					,       A			)
														names.append('''/'''.join(A			)			)
														arrays.append(A			)
							logger.info(F"""Read a total of {len(A			):,} layers"""			)

							# Sanity check
							if len(set(A			)			) != 1:
														raise ValueError(F"""Found layer names with different depths (layer depth {list(set(A			)			)})"""			)
							lowerCAmelCase__			  =      list(set(A			)			)[0]
							if layer_depth != 1:
														raise ValueError(
														    '''The model contains more than just the embedding/encoder layers. This script does not handle MLM/NSP'''
														    ''' heads.'''			)

							# convert layers
							logger.info('''Converting weights...'''			)
							for full_name, array in zip(A					,       A			):
														lowerCAmelCase__			  =      full_name.split('''/'''			)
														lowerCAmelCase__			  =      model
														lowerCAmelCase__			  =      []
														for i, m_name in enumerate(A			):
																					if m_name == ".ATTRIBUTES":
																												# variable names end with .ATTRIBUTES/VARIABLE_VALUE
																												break
																					if m_name.startswith('''layer_with_weights'''			):
																												lowerCAmelCase__			  =      int(m_name.split('''-'''			)[-1]			)
																												if layer_num <= 2:
																																			# embedding layers
																																			# layer_num 0: word_embeddings
																																			# layer_num 1: position_embeddings
																																			# layer_num 2: token_type_embeddings
																																			continue
																												elif layer_num == 3:
																																			# embedding LayerNorm
																																			trace.extend(['''embeddings''', '''LayerNorm''']			)
																																			lowerCAmelCase__			  =      getattr(A					,       '''embeddings'''			)
																																			lowerCAmelCase__			  =      getattr(A					,       '''LayerNorm'''			)
																												elif layer_num > 3 and layer_num < config.num_hidden_layers + 4:
																																			# encoder layers
																																			trace.extend(['''encoder''', '''layer''', str(layer_num - 4			)]			)
																																			lowerCAmelCase__			  =      getattr(A					,       '''encoder'''			)
																																			lowerCAmelCase__			  =      getattr(A					,       '''layer'''			)
																																			lowerCAmelCase__			  =      pointer[layer_num - 4]
																												elif layer_num == config.num_hidden_layers + 4:
																																			# pooler layer
																																			trace.extend(['''pooler''', '''dense''']			)
																																			lowerCAmelCase__			  =      getattr(A					,       '''pooler'''			)
																																			lowerCAmelCase__			  =      getattr(A					,       '''dense'''			)
																					elif m_name == "embeddings":
																												trace.append('''embeddings'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''embeddings'''			)
																												if layer_num == 0:
																																			trace.append('''word_embeddings'''			)
																																			lowerCAmelCase__			  =      getattr(A					,       '''word_embeddings'''			)
																												elif layer_num == 1:
																																			trace.append('''position_embeddings'''			)
																																			lowerCAmelCase__			  =      getattr(A					,       '''position_embeddings'''			)
																												elif layer_num == 2:
																																			trace.append('''token_type_embeddings'''			)
																																			lowerCAmelCase__			  =      getattr(A					,       '''token_type_embeddings'''			)
																												else:
																																			raise ValueError(F"""Unknown embedding layer with name {full_name}"""			)
																												trace.append('''weight'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''weight'''			)
																					elif m_name == "_attention_layer":
																												# self-attention layer
																												trace.extend(['''attention''', '''self''']			)
																												lowerCAmelCase__			  =      getattr(A					,       '''attention'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''self'''			)
																					elif m_name == "_attention_layer_norm":
																												# output attention norm
																												trace.extend(['''attention''', '''output''', '''LayerNorm''']			)
																												lowerCAmelCase__			  =      getattr(A					,       '''attention'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''output'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''LayerNorm'''			)
																					elif m_name == "_attention_output_dense":
																												# output attention dense
																												trace.extend(['''attention''', '''output''', '''dense''']			)
																												lowerCAmelCase__			  =      getattr(A					,       '''attention'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''output'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''dense'''			)
																					elif m_name == "_output_dense":
																												# output dense
																												trace.extend(['''output''', '''dense''']			)
																												lowerCAmelCase__			  =      getattr(A					,       '''output'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''dense'''			)
																					elif m_name == "_output_layer_norm":
																												# output dense
																												trace.extend(['''output''', '''LayerNorm''']			)
																												lowerCAmelCase__			  =      getattr(A					,       '''output'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''LayerNorm'''			)
																					elif m_name == "_key_dense":
																												# attention key
																												trace.append('''key'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''key'''			)
																					elif m_name == "_query_dense":
																												# attention query
																												trace.append('''query'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''query'''			)
																					elif m_name == "_value_dense":
																												# attention value
																												trace.append('''value'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''value'''			)
																					elif m_name == "_intermediate_dense":
																												# attention intermediate dense
																												trace.extend(['''intermediate''', '''dense''']			)
																												lowerCAmelCase__			  =      getattr(A					,       '''intermediate'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''dense'''			)
																					elif m_name == "_output_layer_norm":
																												# output layer norm
																												trace.append('''output'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''output'''			)
																					# weights & biases
																					elif m_name in ["bias", "beta"]:
																												trace.append('''bias'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''bias'''			)
																					elif m_name in ["kernel", "gamma"]:
																												trace.append('''weight'''			)
																												lowerCAmelCase__			  =      getattr(A					,       '''weight'''			)
																					else:
																												logger.warning(F"""Ignored {m_name}"""			)
        # for certain layers reshape is necessary
														lowerCAmelCase__			  =      '''.'''.join(A			)
														if re.match(R'''(\S+)\.attention\.self\.(key|value|query)\.(bias|weight)'''					,       A			) or re.match(
														    R'''(\S+)\.attention\.output\.dense\.weight'''					,       A			):
																					lowerCAmelCase__			  =      array.reshape(pointer.data.shape			)
														if "kernel" in full_name:
																					lowerCAmelCase__			  =      array.transpose()
														if pointer.shape == array.shape:
																					lowerCAmelCase__			  =      torch.from_numpy(A			)
														else:
																					raise ValueError(
																					    F"""Shape mismatch in layer {full_name}: Model expects shape {pointer.shape} but layer contains shape:"""
																					    F""" {array.shape}"""			)
														logger.info(F"""Successfully set variable {full_name} to PyTorch layer {trace}"""			)
							return model


def    _snake_case  (	A					,       A					,       A			)		->       Any:
							# Instantiate model
							logger.info(F"""Loading model based on config from {config_path}..."""			)
							lowerCAmelCase__			  =      BertConfig.from_json_file(A			)
							lowerCAmelCase__			  =      BertModel(A			)

							# Load weights from checkpoint
							logger.info(F"""Loading weights from checkpoint {tf_checkpoint_path}..."""			)
							load_tfa_weights_in_bert(A					,       A					,       A			)

							# Save pytorch-model
							logger.info(F"""Saving PyTorch model to {pytorch_dump_path}..."""			)
							torch.save(model.state_dict()					,       A			)


if __name__ == "__main__":
		__UpperCAmelCase       							=				argparse.ArgumentParser()
		parser.add_argument(
		    '''--tf_checkpoint_path''', type=str, required=True, help='''Path to the TensorFlow 2.x checkpoint path.'''
		)
		parser.add_argument(
		    '''--bert_config_file''',
		    type=str,
		    required=True,
		    help='''The config json file corresponding to the BERT model. This specifies the model architecture.''',
		)
		parser.add_argument(
		    '''--pytorch_dump_path''',
		    type=str,
		    required=True,
		    help='''Path to the output PyTorch model (must include filename).''',
		)
		__UpperCAmelCase       							=				parser.parse_args()
		convert_tfa_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)





'''simple docstring'''







import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase      :   Optional[int]      =       logging.get_logger(__name__)

lowercase      :   Union[str, Any]      =       {
    """microsoft/unispeech-sat-base-100h-libri-ft""": (
        """https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft/resolve/main/config.json"""
    ),
    # See all UniSpeechSat models at https://huggingface.co/models?filter=unispeech_sat
}







class 					_a (a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     List[Any]          =					"""unispeech-sat"""







     def __init__(				self							,__a=32							,__a=768							,__a=12							,__a=12							,__a=3_072							,__a="gelu"							,__a=0.1							,__a=0.1							,__a=0.1							,__a=0.0							,__a=0.0							,__a=0.1							,__a=0.1							,__a=0.02							,__a=1E-5							,__a="group"							,__a="gelu"							,__a=(512, 512, 512, 512, 512, 512, 512)							,__a=(5, 2, 2, 2, 2, 2, 2)							,__a=(10, 3, 3, 3, 3, 2, 2)							,__a=False							,__a=128							,__a=16							,__a=False							,__a=True							,__a=0.05							,__a=10							,__a=2							,__a=0.0							,__a=10							,__a=0							,__a=320							,__a=2							,__a=0.1							,__a=100							,__a=256							,__a=256							,__a=0.1							,__a="mean"							,__a=False							,__a=False							,__a=256							,__a=(512, 512, 512, 512, 1_500)							,__a=(5, 3, 3, 1, 1)							,__a=(1, 2, 3, 1, 1)							,__a=512							,__a=0							,__a=1							,__a=2							,__a=504							,**__a							,)      ->				str:
          super().__init__(**__a							,pad_token_id=__a							,bos_token_id=__a							,eos_token_id=__a							)
          snake_case			:      List[Any]        =			hidden_size
          snake_case			:      Any        =			feat_extract_norm
          snake_case			:      Dict        =			feat_extract_activation
          snake_case			:      List[Any]        =			list(__a							)
          snake_case			:      int        =			list(__a							)
          snake_case			:      Dict        =			list(__a							)
          snake_case			:      int        =			conv_bias
          snake_case			:      Optional[Any]        =			num_conv_pos_embeddings
          snake_case			:      Any        =			num_conv_pos_embedding_groups
          snake_case			:      Dict        =			len(self.conv_dim							)
          snake_case			:      str        =			num_hidden_layers
          snake_case			:      Optional[Any]        =			intermediate_size
          snake_case			:      Optional[Any]        =			hidden_act
          snake_case			:      Union[str, Any]        =			num_attention_heads
          snake_case			:      str        =			hidden_dropout
          snake_case			:      Tuple        =			attention_dropout
          snake_case			:      Tuple        =			activation_dropout
          snake_case			:      List[Any]        =			feat_proj_dropout
          snake_case			:      List[str]        =			final_dropout
          snake_case			:      Union[str, Any]        =			layerdrop
          snake_case			:      Optional[int]        =			layer_norm_eps
          snake_case			:      int        =			initializer_range
          snake_case			:      List[Any]        =			vocab_size
          snake_case			:      Any        =			num_clusters
          snake_case			:      Dict        =			do_stable_layer_norm
          snake_case			:      int        =			use_weighted_layer_sum

          if (
              (len(self.conv_stride							) != self.num_feat_extract_layers)
              or (len(self.conv_kernel							) != self.num_feat_extract_layers)
              or (len(self.conv_dim							) != self.num_feat_extract_layers)
          ):
               raise ValueError(
                   """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
                   """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
                   F''' {len(self.conv_dim							)}`, `len(config.conv_stride) = {len(self.conv_stride							)}`,'''
                   F''' `len(config.conv_kernel) = {len(self.conv_kernel							)}`.'''							)

          # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
          snake_case			:      Dict        =			apply_spec_augment
          snake_case			:      Optional[int]        =			mask_time_prob
          snake_case			:      int        =			mask_time_length
          snake_case			:      int        =			mask_time_min_masks
          snake_case			:      int        =			mask_feature_prob
          snake_case			:      Optional[Any]        =			mask_feature_length
          snake_case			:      List[Any]        =			mask_feature_min_masks

          # parameters for pretraining with codevector quantized representations
          snake_case			:      Optional[Any]        =			num_codevectors_per_group
          snake_case			:      Optional[int]        =			num_codevector_groups
          snake_case			:      Any        =			contrastive_logits_temperature
          snake_case			:      List[str]        =			feat_quantizer_dropout
          snake_case			:      Dict        =			num_negatives
          snake_case			:      int        =			codevector_dim
          snake_case			:      Optional[int]        =			proj_codevector_dim
          snake_case			:      Union[str, Any]        =			diversity_loss_weight

          # ctc loss
          snake_case			:      Union[str, Any]        =			ctc_loss_reduction
          snake_case			:      Optional[int]        =			ctc_zero_infinity

          # SequenceClassification-specific parameter. Feel free to ignore for other classes.
          snake_case			:      Dict        =			classifier_proj_size

          # XVector-specific parameters. Feel free to ignore for other classes.
          snake_case			:      int        =			list(__a							)
          snake_case			:      Tuple        =			list(__a							)
          snake_case			:      List[Any]        =			list(__a							)
          snake_case			:      Dict        =			xvector_output_dim







     @property
     def 			snake_case_      (				self							)      ->				List[str]:
          return functools.reduce(operator.mul							,self.conv_stride							,1							)



