





'''simple docstring'''






# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from ...utils.dataclasses import (
    ComputeEnvironment,
    DistributedType,
    DynamoBackend,
    PrecisionType,
    SageMakerDistributedType,
)
from ..menu import BulletMenu


A_		:  Dict		 =						[
    "EAGER",
    "AOT_EAGER",
    "INDUCTOR",
    "NVFUSER",
    "AOT_NVFUSER",
    "AOT_CUDAGRAPHS",
    "OFI",
    "FX2TRT",
    "ONNXRT",
    "IPEX",
]



def        UpperCamelCase__		(						__magic_name__  :   List[Any]		,       __magic_name__  :   List[Any]=None		,       __magic_name__  :   List[str]=None		,       __magic_name__  :   List[str]=None      )				->			Dict:
 '''simple docstring'''






 snake_case__     :					Optional[int]    =  True
 while ask_again:
  snake_case__     :					Optional[Any]    =  input(__magic_name__      )
  try:
   if default is not None and len(__magic_name__      ) == 0:
    return default
   return convert_value(__magic_name__      ) if convert_value is not None else result
  except Exception:
   if error_message is not None:
    print(__magic_name__      )



def        UpperCamelCase__		(						__magic_name__  :   List[str]		,       __magic_name__  :   Any=[]		,       __magic_name__  :   Optional[int]=None		,       __magic_name__  :   int=0      )				->			Optional[int]:
 '''simple docstring'''






 snake_case__     :					Union[str, Any]    =  BulletMenu(__magic_name__		,       __magic_name__      )
 snake_case__     :					Optional[Any]    =  menu.run(default_choice=__magic_name__      )
 return convert_value(__magic_name__      ) if convert_value is not None else result



def        UpperCamelCase__		(						__magic_name__  :   Any      )				->			int:
 '''simple docstring'''






 snake_case__     :					Tuple    =  int(__magic_name__      )
 return ComputeEnvironment(["""LOCAL_MACHINE""", """AMAZON_SAGEMAKER"""][value]      )



def        UpperCamelCase__		(						__magic_name__  :   str      )				->			Tuple:
 '''simple docstring'''






 snake_case__     :					List[Any]    =  int(__magic_name__      )
 return DistributedType(["""NO""", """MULTI_CPU""", """MULTI_XPU""", """MULTI_GPU""", """MULTI_NPU""", """TPU"""][value]      )



def        UpperCamelCase__		(						__magic_name__  :   List[str]      )				->			List[Any]:
 '''simple docstring'''






 snake_case__     :					Union[str, Any]    =  int(__magic_name__      )
 return DynamoBackend(DYNAMO_BACKENDS[value]      ).value



def        UpperCamelCase__		(						__magic_name__  :   List[str]      )				->			Union[str, Any]:
 '''simple docstring'''






 snake_case__     :					Optional[Any]    =  int(__magic_name__      )
 return PrecisionType(["""no""", """fp16""", """bf16""", """fp8"""][value]      )



def        UpperCamelCase__		(						__magic_name__  :   Optional[int]      )				->			List[Any]:
 '''simple docstring'''






 snake_case__     :					Optional[Any]    =  int(__magic_name__      )
 return SageMakerDistributedType(["""NO""", """DATA_PARALLEL""", """MODEL_PARALLEL"""][value]      )



def        UpperCamelCase__		(						__magic_name__  :   Dict      )				->			Tuple:
 '''simple docstring'''






 return {"yes": True, "no": False}[value.lower()]





class 					__snake_case  (					argparse.RawDescriptionHelpFormatter							):






   '''simple docstring'''





   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       ):
    snake_case__     :					str    =  super()._format_usage(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )
    snake_case__     :					str    =  usage.replace("""<command> [<args>] """	,		""""""       )
    return usage








'''simple docstring'''


import argparse

import torch

from transformers import GPTaLMHeadModel, RobertaForMaskedLM


if __name__ == "__main__":
      _UpperCamelCase 							=  argparse.ArgumentParser(
          description=(
              """Extraction some layers of the full RobertaForMaskedLM or GPT2LMHeadModel for Transfer Learned"""
              """ Distillation"""
          )
      )
      parser.add_argument("""--model_type""", default="""roberta""", choices=["""roberta""", """gpt2"""])
      parser.add_argument("""--model_name""", default="""roberta-large""", type=str)
      parser.add_argument("""--dump_checkpoint""", default="""serialization_dir/tf_roberta_048131723.pth""", type=str)
      parser.add_argument("""--vocab_transform""", action="""store_true""")
      _UpperCamelCase 							=  parser.parse_args()

      if args.model_type == "roberta":
            _UpperCamelCase 							=  RobertaForMaskedLM.from_pretrained(args.model_name)
            _UpperCamelCase 							=  """roberta"""
      elif args.model_type == "gpt2":
            _UpperCamelCase 							=  GPTaLMHeadModel.from_pretrained(args.model_name)
            _UpperCamelCase 							=  """transformer"""

      _UpperCamelCase 							=  model.state_dict()
      _UpperCamelCase 							=  {}

      # Embeddings #
      if args.model_type == "gpt2":
            for param_name in ["wte.weight", "wpe.weight"]:
                  _UpperCamelCase 							=  state_dict[F"""{prefix}.{param_name}"""]
      else:
            for w in ["word_embeddings", "position_embeddings", "token_type_embeddings"]:
                  _UpperCamelCase 							=  F"""{prefix}.embeddings.{w}.weight"""
                  _UpperCamelCase 							=  state_dict[param_name]
            for w in ["weight", "bias"]:
                  _UpperCamelCase 							=  F"""{prefix}.embeddings.LayerNorm.{w}"""
                  _UpperCamelCase 							=  state_dict[param_name]

    # Transformer Blocks #
      _UpperCamelCase 							=  0
      for teacher_idx in [0, 2, 4, 7, 9, 11]:
            if args.model_type == "gpt2":
                  for layer in ["ln_1", "attn.c_attn", "attn.c_proj", "ln_2", "mlp.c_fc", "mlp.c_proj"]:
                        for w in ["weight", "bias"]:
                              _UpperCamelCase 							=  state_dict[
                                  F"""{prefix}.h.{teacher_idx}.{layer}.{w}"""
                              ]
                  _UpperCamelCase 							=  state_dict[F"""{prefix}.h.{teacher_idx}.attn.bias"""]
            else:
                  for layer in [
                      "attention.self.query",
                      "attention.self.key",
                      "attention.self.value",
                      "attention.output.dense",
                      "attention.output.LayerNorm",
                      "intermediate.dense",
                      "output.dense",
                      "output.LayerNorm",
                  ]:
                        for w in ["weight", "bias"]:
                              _UpperCamelCase 							=  state_dict[
                                  F"""{prefix}.encoder.layer.{teacher_idx}.{layer}.{w}"""
                              ]
            std_idx += 1

      # Language Modeling Head ###s
      if args.model_type == "roberta":
            for layer in ["lm_head.decoder.weight", "lm_head.bias"]:
                  _UpperCamelCase 							=  state_dict[F"""{layer}"""]
            if args.vocab_transform:
                  for w in ["weight", "bias"]:
                        _UpperCamelCase 							=  state_dict[F"""lm_head.dense.{w}"""]
                        _UpperCamelCase 							=  state_dict[F"""lm_head.layer_norm.{w}"""]
      elif args.model_type == "gpt2":
            for w in ["weight", "bias"]:
                  _UpperCamelCase 							=  state_dict[F"""{prefix}.ln_f.{w}"""]
            _UpperCamelCase 							=  state_dict["""lm_head.weight"""]

      print(F"""N layers selected for distillation: {std_idx}""")
      print(F"""Number of params transferred for distillation: {len(compressed_sd.keys())}""")

      print(F"""Save transferred checkpoint to {args.dump_checkpoint}.""")
      torch.save(compressed_sd, args.dump_checkpoint)

