

"""simple docstring"""






import argparse
import json
import os

import fairseq
import torch
from fairseq.data import Dictionary

from transformers import (
    HubertConfig,
    HubertForCTC,
    HubertModel,
    WavaVecaCTCTokenizer,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
    logging,
)


logging.set_verbosity_info()
__lowerCamelCase										=		logging.get_logger(__name__)

__lowerCamelCase										=		{
    "post_extract_proj": "feature_projection.projection",
    "encoder.pos_conv.0": "encoder.pos_conv_embed.conv",
    "self_attn.k_proj": "encoder.layers.*.attention.k_proj",
    "self_attn.v_proj": "encoder.layers.*.attention.v_proj",
    "self_attn.q_proj": "encoder.layers.*.attention.q_proj",
    "self_attn.out_proj": "encoder.layers.*.attention.out_proj",
    "self_attn_layer_norm": "encoder.layers.*.layer_norm",
    "fc1": "encoder.layers.*.feed_forward.intermediate_dense",
    "fc2": "encoder.layers.*.feed_forward.output_dense",
    "final_layer_norm": "encoder.layers.*.final_layer_norm",
    "encoder.layer_norm": "encoder.layer_norm",
    "w2v_model.layer_norm": "feature_projection.layer_norm",
    "w2v_encoder.proj": "lm_head",
    "mask_emb": "masked_spec_embed",
}





def 	lowercase      (   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase						) ->      Union[str, Any]:
 for attribute in key.split('''.'''						):
  __magic_name__     =      getattr(__UpperCamelCase      ,   __UpperCamelCase						)

 if weight_type is not None:
  __magic_name__     =      getattr(__UpperCamelCase      ,   __UpperCamelCase						).shape
 else:
  __magic_name__     =      hf_pointer.shape

 assert hf_shape == value.shape, (
     f'''Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be'''
     f''' {value.shape} for {full_name}'''
 )

 if weight_type == "weight":
  __magic_name__     =      value
 elif weight_type == "weight_g":
  __magic_name__     =      value
 elif weight_type == "weight_v":
  __magic_name__     =      value
 elif weight_type == "bias":
  __magic_name__     =      value
 else:
  __magic_name__     =      value

 logger.info(f'''{key + "." + weight_type if weight_type is not None else ""} was initialized from {full_name}.'''						)





def 	lowercase      (   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase						) ->      Dict:
 __magic_name__     =      []
 __magic_name__     =      fairseq_model.state_dict()

 __magic_name__     =      hf_model.hubert.feature_extractor if is_finetuned else hf_model.feature_extractor

 for name, value in fairseq_dict.items():
  __magic_name__     =      False
  if "conv_layers" in name:
   load_conv_layer(
       __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   hf_model.config.feat_extract_norm == '''group'''      ,   )
   __magic_name__     =      True
  else:
   for key, mapped_key in MAPPING.items():
    __magic_name__     =      '''hubert.''' + mapped_key if (is_finetuned and mapped_key != '''lm_head''') else mapped_key

    if key in name or (key.split('''w2v_model.'''						)[-1] == name.split('''.'''						)[0] and not is_finetuned):
     __magic_name__     =      True
     if "*" in mapped_key:
      __magic_name__     =      name.split(__UpperCamelCase						)[0].split('''.'''						)[-2]
      __magic_name__     =      mapped_key.replace('''*'''      ,   __UpperCamelCase						)
     if "weight_g" in name:
      __magic_name__     =      '''weight_g'''
     elif "weight_v" in name:
      __magic_name__     =      '''weight_v'''
     elif "weight" in name:
      __magic_name__     =      '''weight'''
     elif "bias" in name:
      __magic_name__     =      '''bias'''
     else:
      __magic_name__     =      None
     set_recursively(__UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase						)
    continue
  if not is_used:
   unused_weights.append(__UpperCamelCase						)

 logger.warning(f'''Unused weights: {unused_weights}'''						)





def 	lowercase      (   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase						) ->      Optional[int]:
 __magic_name__     =      full_name.split('''conv_layers.'''						)[-1]
 __magic_name__     =      name.split('''.'''						)
 __magic_name__     =      int(items[0]						)
 __magic_name__     =      int(items[1]						)

 if type_id == 0:
  if "bias" in name:
   assert value.shape == feature_extractor.conv_layers[layer_id].conv.bias.data.shape, (
       f'''{full_name} has size {value.shape}, but'''
       f''' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'''
   )
   __magic_name__     =      value
   logger.info(f'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''						)
  elif "weight" in name:
   assert value.shape == feature_extractor.conv_layers[layer_id].conv.weight.data.shape, (
       f'''{full_name} has size {value.shape}, but'''
       f''' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'''
   )
   __magic_name__     =      value
   logger.info(f'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''						)
 elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
  if "bias" in name:
   assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape, (
       f'''{full_name} has size {value.shape}, but {feature_extractor[layer_id].layer_norm.bias.data.shape} was'''
       " found."
   )
   __magic_name__     =      value
   logger.info(f'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''						)
  elif "weight" in name:
   assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape, (
       f'''{full_name} has size {value.shape}, but'''
       f''' {feature_extractor[layer_id].layer_norm.weight.data.shape} was found.'''
   )
   __magic_name__     =      value
   logger.info(f'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''						)
 else:
  unused_weights.append(__UpperCamelCase						)





@torch.no_grad()
def 	lowercase      (   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase=None      ,   __UpperCamelCase=None      ,   __UpperCamelCase=True						) ->      Any:
 if config_path is not None:
  __magic_name__     =      HubertConfig.from_pretrained(__UpperCamelCase						)
 else:
  __magic_name__     =      HubertConfig()

 if is_finetuned:
  if dict_path:
   __magic_name__     =      Dictionary.load(__UpperCamelCase						)

   # important change bos & pad token id since CTC symbol is <pad> and
   # not <s> as in fairseq
   __magic_name__     =      target_dict.pad_index
   __magic_name__     =      target_dict.bos_index
   __magic_name__     =      target_dict.eos_index
   __magic_name__     =      len(target_dict.symbols						)
   __magic_name__     =      os.path.join(__UpperCamelCase      ,   '''vocab.json'''						)
   if not os.path.isdir(__UpperCamelCase						):
    logger.error('''--pytorch_dump_folder_path ({}) should be a directory'''.format(__UpperCamelCase						)						)
    return
   os.makedirs(__UpperCamelCase      ,   exist_ok=__UpperCamelCase						)
   with open(__UpperCamelCase      ,   '''w'''      ,   encoding='''utf-8'''						) as vocab_handle:
    json.dump(target_dict.indices      ,   __UpperCamelCase						)
   __magic_name__     =      WavaVecaCTCTokenizer(
       __UpperCamelCase      ,   unk_token=target_dict.unk_word      ,   pad_token=target_dict.pad_word      ,   bos_token=target_dict.bos_word      ,   eos_token=target_dict.eos_word      ,   word_delimiter_token='''|'''      ,   do_lower_case=__UpperCamelCase      ,   )
   __magic_name__     =      True if config.feat_extract_norm == '''layer''' else False
   __magic_name__     =      WavaVecaFeatureExtractor(
       feature_size=1      ,   sampling_rate=16000      ,   padding_value=0      ,   do_normalize=__UpperCamelCase      ,   return_attention_mask=__UpperCamelCase      ,   )
   __magic_name__     =      WavaVecaProcessor(feature_extractor=__UpperCamelCase      ,   tokenizer=__UpperCamelCase						)
   processor.save_pretrained(__UpperCamelCase						)

  __magic_name__     =      HubertForCTC(__UpperCamelCase						)
 else:
  __magic_name__     =      HubertModel(__UpperCamelCase						)

 if is_finetuned:
  __magic_name__   ,						__magic_name__   ,						__magic_name__     =      fairseq.checkpoint_utils.load_model_ensemble_and_task(
      [checkpoint_path]      ,   arg_overrides={'''data''': '''/'''.join(dict_path.split('''/'''						)[:-1]						)}						)
 else:
  __magic_name__   ,						__magic_name__   ,						__magic_name__     =      fairseq.checkpoint_utils.load_model_ensemble_and_task([checkpoint_path]						)

 __magic_name__     =      model[0].eval()

 recursively_load_weights(__UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase						)

 hf_wavavec.save_pretrained(__UpperCamelCase						)


if __name__ == "__main__":
 __lowerCamelCase										=		argparse.ArgumentParser()
 parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
 parser.add_argument("--checkpoint_path", default=None, type=str, help="Path to fairseq checkpoint")
 parser.add_argument("--dict_path", default=None, type=str, help="Path to dict of fine-tuned model")
 parser.add_argument("--config_path", default=None, type=str, help="Path to hf config.json of model to convert")
 parser.add_argument(
     "--not_finetuned", action="store_true", help="Whether the model to convert is a fine-tuned model or not"
 )
 __lowerCamelCase										=		parser.parse_args()
 convert_hubert_checkpoint(
     args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, not args.not_finetuned
 )




"""simple docstring"""






import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


__lowerCamelCase										=		"hf-internal-testing/tiny-random-bert"
__lowerCamelCase										=		os.path.join(TRANSFORMERS_CACHE, "models--hf-internal-testing--tiny-random-bert")
__lowerCamelCase										=		"9b8c223d42b2188cb49d29af482996f9d0f3e5a6"
class        _lowercase					(     unittest.TestCase     ):







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      cached_file(UpperCamelCase_ ,						UpperCamelCase_       )
      # Should have downloaded the file in here
      self.assertTrue(os.path.isdir(UpperCamelCase_       )       )
      # Cache should contain at least those three subfolders:
      for subfolder in ["blobs", "refs", "snapshots"]:
       self.assertTrue(os.path.isdir(os.path.join(UpperCamelCase_ ,						UpperCamelCase_       )       )       )
      with open(os.path.join(UpperCamelCase_ ,						'''refs''' ,						'''main'''       )       ) as f:
       __magic_name__     =      f.read()
      self.assertEqual(UpperCamelCase_ ,						os.path.join(UpperCamelCase_ ,						'''snapshots''' ,						UpperCamelCase_ ,						UpperCamelCase_       )       )
      self.assertTrue(os.path.isfile(UpperCamelCase_       )       )

      # File is cached at the same place the second time.
      __magic_name__     =      cached_file(UpperCamelCase_ ,						UpperCamelCase_       )
      self.assertEqual(UpperCamelCase_ ,						UpperCamelCase_       )

      # Using a specific revision to test the full commit hash.
      __magic_name__     =      cached_file(UpperCamelCase_ ,						UpperCamelCase_ ,						revision='''9b8c223'''       )
      self.assertEqual(UpperCamelCase_ ,						os.path.join(UpperCamelCase_ ,						'''snapshots''' ,						UpperCamelCase_ ,						UpperCamelCase_       )       )







     def 	lowerCAmelCase__    (      self       ):
      with self.assertRaisesRegex(UpperCamelCase_ ,						'''is not a valid model identifier'''       ):
       __magic_name__     =      cached_file('''tiny-random-bert''' ,						UpperCamelCase_       )

      with self.assertRaisesRegex(UpperCamelCase_ ,						'''is not a valid git identifier'''       ):
       __magic_name__     =      cached_file(UpperCamelCase_ ,						UpperCamelCase_ ,						revision='''aaaa'''       )

      with self.assertRaisesRegex(UpperCamelCase_ ,						'''does not appear to have a file named'''       ):
       __magic_name__     =      cached_file(UpperCamelCase_ ,						'''conf'''       )







     def 	lowerCAmelCase__    (      self       ):
      with self.assertRaisesRegex(UpperCamelCase_ ,						'''does not appear to have a file named'''       ):
       __magic_name__     =      cached_file(UpperCamelCase_ ,						'''conf'''       )

      with open(os.path.join(UpperCamelCase_ ,						'''refs''' ,						'''main'''       )       ) as f:
       __magic_name__     =      f.read()
      self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase_ ,						'''.no_exist''' ,						UpperCamelCase_ ,						'''conf'''       )       )       )

      __magic_name__     =      cached_file(UpperCamelCase_ ,						'''conf''' ,						_raise_exceptions_for_missing_entries=UpperCamelCase_       )
      self.assertIsNone(UpperCamelCase_       )

      __magic_name__     =      cached_file(UpperCamelCase_ ,						'''conf''' ,						local_files_only=UpperCamelCase_ ,						_raise_exceptions_for_missing_entries=UpperCamelCase_       )
      self.assertIsNone(UpperCamelCase_       )

      __magic_name__     =      mock.Mock()
      __magic_name__     =      500
      __magic_name__     =      {}
      __magic_name__     =      HTTPError
      __magic_name__     =      {}

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request''' ,						return_value=UpperCamelCase_       ) as mock_head:
       __magic_name__     =      cached_file(UpperCamelCase_ ,						'''conf''' ,						_raise_exceptions_for_connection_errors=UpperCamelCase_       )
       self.assertIsNone(UpperCamelCase_       )
       # This check we did call the fake head request
       mock_head.assert_called()







     def 	lowerCAmelCase__    (      self       ):
      self.assertTrue(has_file('''hf-internal-testing/tiny-bert-pt-only''' ,						UpperCamelCase_       )       )
      self.assertFalse(has_file('''hf-internal-testing/tiny-bert-pt-only''' ,						UpperCamelCase_       )       )
      self.assertFalse(has_file('''hf-internal-testing/tiny-bert-pt-only''' ,						UpperCamelCase_       )       )







     def 	lowerCAmelCase__    (      self       ):
      # `get_file_from_repo` returns None if the file does not exist
      self.assertIsNone(get_file_from_repo('''bert-base-cased''' ,						'''ahah.txt'''       )       )

      # The function raises if the repository does not exist.
      with self.assertRaisesRegex(UpperCamelCase_ ,						'''is not a valid model identifier'''       ):
       get_file_from_repo('''bert-base-case''' ,						UpperCamelCase_       )

      # The function raises if the revision does not exist.
      with self.assertRaisesRegex(UpperCamelCase_ ,						'''is not a valid git identifier'''       ):
       get_file_from_repo('''bert-base-cased''' ,						UpperCamelCase_ ,						revision='''ahaha'''       )

      __magic_name__     =      get_file_from_repo('''bert-base-cased''' ,						UpperCamelCase_       )
      # The name is the cached name which is not very easy to test, so instead we load the content.
      __magic_name__     =      json.loads(open(UpperCamelCase_ ,						'''r'''       ).read()       )
      self.assertEqual(config['''hidden_size'''] ,						768       )







     def 	lowerCAmelCase__    (      self       ):
      with tempfile.TemporaryDirectory() as tmp_dir:
       __magic_name__     =      Path(UpperCamelCase_       ) / '''a.txt'''
       filename.touch()
       self.assertEqual(get_file_from_repo(UpperCamelCase_ ,						'''a.txt'''       ) ,						str(UpperCamelCase_       )       )

       self.assertIsNone(get_file_from_repo(UpperCamelCase_ ,						'''b.txt'''       )       )

