





from manim import *

class       __A     (						a    ):


				def _snake_case       (		self							):
								lowerCamelCase	 =Rectangle(height=0.5   , width=0.5							)
								lowerCamelCase	 =Rectangle(height=0.4_6   , width=0.4_6							).set_stroke(width=0							)

								lowerCamelCase	 =Rectangle(height=0.2_5   , width=0.2_5							)

								lowerCamelCase	 =[mem.copy() for i in range(6							)]
								lowerCamelCase	 =[mem.copy() for i in range(6							)]
								lowerCamelCase	 =VGroup(*UpperCAmelCase_							).arrange(UpperCAmelCase_   , buff=0							)
								lowerCamelCase	 =VGroup(*UpperCAmelCase_							).arrange(UpperCAmelCase_   , buff=0							)
								lowerCamelCase	 =VGroup(UpperCAmelCase_   , UpperCAmelCase_							).arrange(UpperCAmelCase_   , buff=0							)
								lowerCamelCase	 =Text("""CPU"""   , font_size=24							)
								lowerCamelCase	 =Group(UpperCAmelCase_   , UpperCAmelCase_							).arrange(UpperCAmelCase_   , buff=0.5   , aligned_edge=UpperCAmelCase_							)
								cpu.move_to([-2.5, -0.5, 0]							)
								self.add(UpperCAmelCase_							)

								lowerCamelCase	 =[mem.copy() for i in range(4							)]
								lowerCamelCase	 =VGroup(*UpperCAmelCase_							).arrange(UpperCAmelCase_   , buff=0							)
								lowerCamelCase	 =Text("""GPU"""   , font_size=24							)
								lowerCamelCase	 =Group(UpperCAmelCase_   , UpperCAmelCase_							).arrange(UpperCAmelCase_   , buff=0.5   , aligned_edge=UpperCAmelCase_							)
								gpu.move_to([-1, -1, 0]							)
								self.add(UpperCAmelCase_							)

								lowerCamelCase	 =[mem.copy() for i in range(6							)]
								lowerCamelCase	 =VGroup(*UpperCAmelCase_							).arrange(UpperCAmelCase_   , buff=0							)

								lowerCamelCase	 =Text("""Model"""   , font_size=24							)
								lowerCamelCase	 =Group(UpperCAmelCase_   , UpperCAmelCase_							).arrange(UpperCAmelCase_   , buff=0.5   , aligned_edge=UpperCAmelCase_							)
								model.move_to([3, -1.0, 0]							)
								self.add(UpperCAmelCase_							)

								lowerCamelCase	 =[]
								lowerCamelCase	 =[]

								for i, rect in enumerate(UpperCAmelCase_							):
												lowerCamelCase	 =fill.copy().set_fill(UpperCAmelCase_   , opacity=0.8							)
												target.move_to(UpperCAmelCase_							)
												model_arr.append(UpperCAmelCase_							)

												lowerCamelCase	 =Rectangle(height=0.4_6   , width=0.4_6							).set_stroke(width=0.0							).set_fill(UpperCAmelCase_   , opacity=0.8							)
												cpu_target.move_to(cpu_left_col_base[i]							)
												model_cpu_arr.append(UpperCAmelCase_							)

								self.add(*UpperCAmelCase_   , *UpperCAmelCase_							)

								lowerCamelCase	 =[meta_mem.copy() for i in range(6							)]
								lowerCamelCase	 =[meta_mem.copy() for i in range(6							)]
								lowerCamelCase	 =VGroup(*UpperCAmelCase_							).arrange(UpperCAmelCase_   , buff=0							)
								lowerCamelCase	 =VGroup(*UpperCAmelCase_							).arrange(UpperCAmelCase_   , buff=0							)
								lowerCamelCase	 =VGroup(UpperCAmelCase_   , UpperCAmelCase_							).arrange(UpperCAmelCase_   , buff=0							)
								lowerCamelCase	 =Text("""Disk"""   , font_size=24							)
								lowerCamelCase	 =Group(UpperCAmelCase_   , UpperCAmelCase_							).arrange(UpperCAmelCase_   , buff=0.5   , aligned_edge=UpperCAmelCase_							)
								disk.move_to([-4, -1.2_5, 0]							)
								self.add(UpperCAmelCase_   , UpperCAmelCase_							)

								lowerCamelCase	 =Square(side_length=2.2							)
								key.move_to([-5, 2, 0]							)

								lowerCamelCase	 =MarkupText(
								    f"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"""   , font_size=18   , )

								key_text.move_to([-5, 2.4, 0]							)

								self.add(UpperCAmelCase_   , UpperCAmelCase_							)

								lowerCamelCase	 =MarkupText(
								    f"""<span fgcolor='{BLUE}'>●</span> Checkpoint"""   , font_size=18   , )

								blue_text.next_to(UpperCAmelCase_   , DOWN * 2.4   , aligned_edge=key_text.get_left()							)
								self.add(UpperCAmelCase_							)

								lowerCamelCase	 =MarkupText(
								    f"""Now watch as an input is passed through the model\nand how the memory is utilized and handled."""   , font_size=24   , )
								step_a.move_to([2, 2, 0]							)

								self.play(Write(UpperCAmelCase_							)							)

								lowerCamelCase	 =Square(0.3							)
								input.set_fill(UpperCAmelCase_   , opacity=1.0							)
								input.set_stroke(width=0.0							)
								input.next_to(model_base[0]   , UpperCAmelCase_   , buff=0.5							)

								self.play(Write(UpperCAmelCase_							)							)

								input.generate_target()
								input.target.next_to(model_arr[0]   , direction=UpperCAmelCase_   , buff=0.0_2							)
								self.play(MoveToTarget(UpperCAmelCase_							)							)

								self.play(FadeOut(UpperCAmelCase_							)							)

								lowerCamelCase	 =Arrow(start=UpperCAmelCase_   , end=UpperCAmelCase_   , color=UpperCAmelCase_   , buff=0.5							)
								a.next_to(model_arr[0].get_left()   , UpperCAmelCase_   , buff=0.2							)

								model_cpu_arr[0].generate_target()
								model_cpu_arr[0].target.move_to(gpu_rect[0]							)

								lowerCamelCase	 =MarkupText(
								    f"""As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back."""   , font_size=24   , )
								step_a.move_to([2, 2, 0]							)

								self.play(Write(UpperCAmelCase_   , run_time=3							)							)

								lowerCamelCase	 ={"""run_time""": 1, """fade_in""": True, """fade_out""": True, """buff""": 0.0_2}

								self.play(
								    Write(UpperCAmelCase_							)   , Circumscribe(model_arr[0]   , color=UpperCAmelCase_   , **UpperCAmelCase_							)   , Circumscribe(model_cpu_arr[0]   , color=UpperCAmelCase_   , **UpperCAmelCase_							)   , Circumscribe(gpu_rect[0]   , color=UpperCAmelCase_   , **UpperCAmelCase_							)   , )
								self.play(MoveToTarget(model_cpu_arr[0]							)							)

								lowerCamelCase	 =a.copy()
								for i in range(6							):
												a_c.next_to(model_arr[i].get_right() + 0.0_2   , UpperCAmelCase_   , buff=0.2							)

												input.generate_target()
												input.target.move_to(model_arr[i].get_right() + 0.0_2							)

												lowerCamelCase	 =AnimationGroup(
												    FadeOut(UpperCAmelCase_   , run_time=0.5							)   , MoveToTarget(UpperCAmelCase_   , run_time=0.5							)   , FadeIn(UpperCAmelCase_   , run_time=0.5							)   , lag_ratio=0.2							)

												self.play(UpperCAmelCase_							)

												model_cpu_arr[i].generate_target()
												model_cpu_arr[i].target.move_to(cpu_left_col_base[i]							)

												if i < 5:
																model_cpu_arr[i + 1].generate_target()
																model_cpu_arr[i + 1].target.move_to(gpu_rect[0]							)
																if i >= 1:
																				lowerCamelCase	 =0.7

																self.play(
																    Circumscribe(model_arr[i]   , **UpperCAmelCase_							)   , Circumscribe(cpu_left_col_base[i]   , **UpperCAmelCase_							)   , Circumscribe(cpu_left_col_base[i + 1]   , color=UpperCAmelCase_   , **UpperCAmelCase_							)   , Circumscribe(gpu_rect[0]   , color=UpperCAmelCase_   , **UpperCAmelCase_							)   , Circumscribe(model_arr[i + 1]   , color=UpperCAmelCase_   , **UpperCAmelCase_							)   , )
																if i < 1:
																				self.play(
																				    MoveToTarget(model_cpu_arr[i]							)   , MoveToTarget(model_cpu_arr[i + 1]							)   , )
																else:
																				self.play(
																				    MoveToTarget(model_cpu_arr[i]   , run_time=0.7							)   , MoveToTarget(model_cpu_arr[i + 1]   , run_time=0.7							)   , )
												else:
																model_cpu_arr[i].generate_target()
																model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]							)
																input.generate_target()
																input.target.next_to(model_arr[-1].get_right()   , RIGHT + 0.0_2   , buff=0.2							)

																self.play(
																    Circumscribe(model_arr[-1]   , color=UpperCAmelCase_   , **UpperCAmelCase_							)   , Circumscribe(cpu_left_col_base[-1]   , color=UpperCAmelCase_   , **UpperCAmelCase_							)   , Circumscribe(gpu_rect[0]   , color=UpperCAmelCase_   , **UpperCAmelCase_							)   , )

																self.play(MoveToTarget(model_cpu_arr[i]							)							)

												lowerCamelCase	 =a_c
												lowerCamelCase	 =a_c.copy()

								input.generate_target()
								input.target.next_to(model_base[-1]   , RIGHT + 0.0_2   , buff=0.5							)
								self.play(
								    FadeOut(UpperCAmelCase_							)   , FadeOut(UpperCAmelCase_   , run_time=0.5							)   , )

								lowerCamelCase	 =MarkupText(f"""Inference on a model too large for GPU memory\nis successfully completed."""   , font_size=24							)
								step_a.move_to([2, 2, 0]							)

								self.play(Write(UpperCAmelCase_   , run_time=3							)   , MoveToTarget(UpperCAmelCase_							)							)

								self.wait()












import argparse
from pathlib import Path
from typing import Dict, OrderedDict, Tuple

import torch
from audiocraft.models import MusicGen

from transformers import (
    AutoFeatureExtractor,
    AutoTokenizer,
    EncodecModel,
    MusicgenDecoderConfig,
    MusicgenForConditionalGeneration,
    MusicgenProcessor,
    TaEncoderModel,
)
from transformers.models.musicgen.modeling_musicgen import MusicgenForCausalLM
from transformers.utils import logging


logging.set_verbosity_info()
UpperCAmelCase__						:			Tuple									=logging.get_logger(__name__)


UpperCAmelCase__						:			str									=['''model.decoder.embed_positions.weights''']





def      _lowercase					(						_UpperCAmelCase				)						->  List[Any]:
				if "emb" in name:
								lowerCamelCase	 =name.replace("""emb"""			,      """model.decoder.embed_tokens"""				)
				if "transformer" in name:
								lowerCamelCase	 =name.replace("""transformer"""			,      """model.decoder"""				)
				if "cross_attention" in name:
								lowerCamelCase	 =name.replace("""cross_attention"""			,      """encoder_attn"""				)
				if "linear1" in name:
								lowerCamelCase	 =name.replace("""linear1"""			,      """fc1"""				)
				if "linear2" in name:
								lowerCamelCase	 =name.replace("""linear2"""			,      """fc2"""				)
				if "norm1" in name:
								lowerCamelCase	 =name.replace("""norm1"""			,      """self_attn_layer_norm"""				)
				if "norm_cross" in name:
								lowerCamelCase	 =name.replace("""norm_cross"""			,      """encoder_attn_layer_norm"""				)
				if "norm2" in name:
								lowerCamelCase	 =name.replace("""norm2"""			,      """final_layer_norm"""				)
				if "out_norm" in name:
								lowerCamelCase	 =name.replace("""out_norm"""			,      """model.decoder.layer_norm"""				)
				if "linears" in name:
								lowerCamelCase	 =name.replace("""linears"""			,      """lm_heads"""				)
				if "condition_provider.conditioners.description.output_proj" in name:
								lowerCamelCase	 =name.replace("""condition_provider.conditioners.description.output_proj"""			,      """enc_to_dec_proj"""				)
				return name





def      _lowercase					(						_UpperCAmelCase			,      _UpperCAmelCase				)						->  Tuple[Dict, Dict]:
				lowerCamelCase	 =list(state_dict.keys()				)
				lowerCamelCase	 ={}
				for key in keys:
								lowerCamelCase	 =state_dict.pop(_UpperCAmelCase				)
								lowerCamelCase	 =rename_keys(_UpperCAmelCase				)
								if "in_proj_weight" in key:
												# split fused qkv proj
												lowerCamelCase	 =val[:hidden_size, :]
												lowerCamelCase	 =val[hidden_size : 2 * hidden_size, :]
												lowerCamelCase	 =val[-hidden_size:, :]
								elif "enc_to_dec_proj" in key:
												lowerCamelCase	 =val
								else:
												lowerCamelCase	 =val
				return state_dict, enc_dec_proj_state_dict





def      _lowercase					(						_UpperCAmelCase				)						->  MusicgenDecoderConfig:
				if checkpoint == "small":
								# default config values
								lowerCamelCase	 =10_24
								lowerCamelCase	 =24
								lowerCamelCase	 =16
				elif checkpoint == "medium":
								lowerCamelCase	 =15_36
								lowerCamelCase	 =48
								lowerCamelCase	 =24
				elif checkpoint == "large":
								lowerCamelCase	 =20_48
								lowerCamelCase	 =48
								lowerCamelCase	 =32
				else:
								raise ValueError(F"""Checkpoint should be one of `['small', 'medium', 'large']`, got {checkpoint}."""				)
				lowerCamelCase	 =MusicgenDecoderConfig(
				    hidden_size=_UpperCAmelCase			,      ffn_dim=hidden_size * 4			,      num_hidden_layers=_UpperCAmelCase			,      num_attention_heads=_UpperCAmelCase			,      )
				return config





@torch.no_grad()
def      _lowercase					(						_UpperCAmelCase			,      _UpperCAmelCase=None			,      _UpperCAmelCase=None			,      _UpperCAmelCase="cpu"				)						->  Dict:
				lowerCamelCase	 =MusicGen.get_pretrained(_UpperCAmelCase			,      device=_UpperCAmelCase				)
				lowerCamelCase	 =decoder_config_from_checkpoint(_UpperCAmelCase				)

				lowerCamelCase	 =fairseq_model.lm.state_dict()
				lowerCamelCase ,  lowerCamelCase	 =rename_state_dict(
				    _UpperCAmelCase			,      hidden_size=decoder_config.hidden_size				)

				lowerCamelCase	 =TaEncoderModel.from_pretrained("""t5-base"""				)
				lowerCamelCase	 =EncodecModel.from_pretrained("""facebook/encodec_32khz"""				)
				lowerCamelCase	 =MusicgenForCausalLM(_UpperCAmelCase				).eval()

				# load all decoder weights - expect that we'll be missing embeddings and enc-dec projection
				lowerCamelCase ,  lowerCamelCase	 =decoder.load_state_dict(_UpperCAmelCase			,      strict=_UpperCAmelCase				)

				for key in missing_keys.copy():
								if key.startswith(("""text_encoder""", """audio_encoder""")				) or key in EXPECTED_MISSING_KEYS:
												missing_keys.remove(_UpperCAmelCase				)

				if len(_UpperCAmelCase				) > 0:
								raise ValueError(F"""Missing key(s) in state_dict: {missing_keys}"""				)

				if len(_UpperCAmelCase				) > 0:
								raise ValueError(F"""Unexpected key(s) in state_dict: {unexpected_keys}"""				)

				# init the composite model
				lowerCamelCase	 =MusicgenForConditionalGeneration(text_encoder=_UpperCAmelCase			,      audio_encoder=_UpperCAmelCase			,      decoder=_UpperCAmelCase				)

				# load the pre-trained enc-dec projection (from the decoder state dict)
				model.enc_to_dec_proj.load_state_dict(_UpperCAmelCase				)

				# check we can do a forward pass
				lowerCamelCase	 =torch.arange(0			,      8			,      dtype=torch.long				).reshape(2			,      -1				)
				lowerCamelCase	 =input_ids.reshape(2 * 4			,      -1				)

				with torch.no_grad():
								lowerCamelCase	 =model(input_ids=_UpperCAmelCase			,      decoder_input_ids=_UpperCAmelCase				).logits

				if logits.shape != (8, 1, 20_48):
								raise ValueError("""Incorrect shape for logits"""				)

				# now construct the processor
				lowerCamelCase	 =AutoTokenizer.from_pretrained("""t5-base"""				)
				lowerCamelCase	 =AutoFeatureExtractor.from_pretrained("""facebook/encodec_32khz"""			,      padding_side="""left"""				)

				lowerCamelCase	 =MusicgenProcessor(feature_extractor=_UpperCAmelCase			,      tokenizer=_UpperCAmelCase				)

				# set the appropriate bos/pad token ids
				lowerCamelCase	 =20_48
				lowerCamelCase	 =20_48

				# set other default generation config params
				lowerCamelCase	 =int(30 * audio_encoder.config.frame_rate				)
				lowerCamelCase	 =True
				lowerCamelCase	 =3.0

				if pytorch_dump_folder is not None:
								Path(_UpperCAmelCase				).mkdir(exist_ok=_UpperCAmelCase				)
								logger.info(F"""Saving model {checkpoint} to {pytorch_dump_folder}"""				)
								model.save_pretrained(_UpperCAmelCase				)
								processor.save_pretrained(_UpperCAmelCase				)

				if repo_id:
								logger.info(F"""Pushing model {checkpoint} to {repo_id}"""				)
								model.push_to_hub(_UpperCAmelCase				)
								processor.push_to_hub(_UpperCAmelCase				)


if __name__ == "__main__":
							UpperCAmelCase__						:			List[Any]									=argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '''--checkpoint''',
							    default='''small''',
							    type=str,
							    help='''Checkpoint size of the MusicGen model you\'d like to convert. Can be one of: `[\'small\', \'medium\', \'large\']`.''',
							)
							parser.add_argument(
							    '''--pytorch_dump_folder''',
							    required=True,
							    default=None,
							    type=str,
							    help='''Path to the output PyTorch model directory.''',
							)
							parser.add_argument(
							    '''--push_to_hub''', default=None, type=str, help='''Where to upload the converted model on the 🤗 hub.'''
							)
							parser.add_argument(
							    '''--device''', default='''cpu''', type=str, help='''Torch device to run the conversion, either cpu or cuda.'''
							)

							UpperCAmelCase__						:			Optional[Any]									=parser.parse_args()
							convert_musicgen_checkpoint(args.checkpoint, args.pytorch_dump_folder, args.push_to_hub)





