





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


SCREAMING_SNAKE_CASE_   :      int			=   logging.getLogger(__name__)



def  SCREAMING_SNAKE_CASE				(      )		->    Optional[Any]:
		__lowercase											=     argparse.ArgumentParser(
		    description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'							)
		parser.add_argument(
		    '--dataset_name'						,   type=snake_case						,   default='wikitext'						,   help='Name of the training. Explore datasets at: hf.co/datasets.'						,   )
		parser.add_argument(
		    '--dataset_config'						,   type=snake_case						,   default='wikitext-103-raw-v1'						,   help='Configuration name of the dataset.'							)
		parser.add_argument(
		    '--tokenizer_name_or_path'						,   type=snake_case						,   default='sayakpaul/unigram-tokenizer-wikitext'						,   help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'						,   )
		parser.add_argument(
		    '--shard_size'						,   type=snake_case						,   default=1_000						,   help='Number of entries to go in a single shard.'						,   )
		parser.add_argument('--split'						,   type=snake_case						,   default='train'						,   choices=['train', 'test', 'validation']							)
		parser.add_argument(
		    '--limit'						,   default=snake_case						,   type=snake_case						,   help='Limit the number of shards (used for debugging).'						,   )
		parser.add_argument(
		    '--max_length'						,   type=snake_case						,   default=512						,   help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
		    ' sequence length that is a multiple of 8.'						,   )
		parser.add_argument(
		    '--output_dir'						,   default='tf-tpu'						,   type=snake_case						,   help='Output directory where the TFRecord shards will be saved. If the'
		    ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
		    ' shards will be directly saved to a Google Cloud Storage bucket.'						,   )

		__lowercase											=     parser.parse_args()
		return args



def  SCREAMING_SNAKE_CASE				(      snake_case							)		->    Union[str, Any]:
		def fn(snake_case							):
				return tokenizer(examples['text']							)

		return fn



def  SCREAMING_SNAKE_CASE				(      snake_case							)		->    List[Any]:
		__lowercase											=     []
		for i in range(len(tokenized_data['input_ids']							)							):
				__lowercase											=     {
				    'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]							)							),
				    'attention_mask': tf.train.Feature(
				        intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]							)							),
				}
				__lowercase											=     tf.train.Features(feature=snake_case							)
				__lowercase											=     tf.train.Example(features=snake_case							)
				__lowercase											=     example.SerializeToString()
				records.append(snake_case							)
		return records



def  SCREAMING_SNAKE_CASE				(      snake_case							)		->    int:
		__lowercase											=     datasets.load_dataset(args.dataset_name						,   args.dataset_config						,   split=args.split							)

		if args.limit is not None:
				__lowercase											=     min(len(snake_case							)						,   args.limit							)
				__lowercase											=     dataset.select(range(snake_case							)							)
				print(F"Limiting the dataset to {args.limit} entries."							)

		__lowercase											=     AutoTokenizer.from_pretrained(args.tokenizer_name_or_path							)

		# Handle output directory creation.
		# For serializing into a Google Cloud Storage Bucket, one needs to first
		# create a bucket.
		if "gs" not in args.output_dir:
				if not os.path.exists(args.output_dir							):
						os.makedirs(args.output_dir							)
				__lowercase											=     os.path.join(args.output_dir						,   args.split							)
				if not os.path.exists(snake_case							):
						os.makedirs(snake_case							)
		else:
				__lowercase											=     os.path.join(args.output_dir						,   args.split							)

		# Tokenize the whole dataset at once.
		__lowercase											=     tokenize_function(snake_case							)
		__lowercase											=     dataset.map(snake_case						,   batched=snake_case						,   num_proc=4						,   remove_columns=['text']							)

		# We need to concatenate all our texts together, and then split the result
		# into chunks of a fixed size, which we will call block_size. To do this, we
		# will use the map method again, with the option batched=True. When we use batched=True,
		# the function we pass to map() will be passed multiple inputs at once, allowing us
		# to group them into more or fewer examples than we had in the input.
		# This allows us to create our new fixed-length samples. The advantage of this
		# method is that we don't lose a whole lot of content from the dataset compared to the
		# case where we simply tokenize with a pre-defined max_length.

		def group_texts(snake_case							):
				# Concatenate all texts.
				__lowercase											=     {k: sum(examples[k]						,   []							) for k in examples.keys()}
				__lowercase											=     len(concatenated_examples[list(examples.keys()							)[0]]							)
				# We drop the small remainder, though you could add padding instead if the model supports it
				# In this, as in all things, we advise you to follow your heart 🫀
				__lowercase											=     (total_length // args.max_length) * args.max_length
				# Split by chunks of max_len.
				__lowercase											=     {
				    k: [t[i : i + args.max_length] for i in range(0						,   snake_case						,   args.max_length							)]
				    for k, t in concatenated_examples.items()
				}
				return result

		__lowercase											=     dataset_tokenized.map(snake_case						,   batched=snake_case						,   batch_size=1_000						,   num_proc=4							)

		__lowercase											=     0
		__lowercase											=     0
		for shard in range(0						,   len(snake_case							)						,   args.shard_size							):
				__lowercase											=     grouped_dataset[shard : shard + args.shard_size]
				__lowercase											=     len(dataset_snapshot['input_ids']							)
				__lowercase											=     os.path.join(snake_case						,   F"dataset-{shard_count}-{records_containing}.tfrecord"							)
				__lowercase											=     get_serialized_examples(snake_case							)

				with tf.io.TFRecordWriter(snake_case							) as out_file:
						for i in range(len(snake_case							)							):
								__lowercase											=     serialized_examples[i]
								out_file.write(snake_case							)
						print('Wrote file {} containing {} records'.format(snake_case						,   snake_case							)							)

				shard_count += 1
				total_records += records_containing

		with open(F"split-{args.split}-records-count.txt"						,   'w'							) as f:
				print(F"Total {args.split} records: {total_records}"						,   file=snake_case							)


if __name__ == "__main__":
		SCREAMING_SNAKE_CASE_   :      List[str]			=   parse_args()
		main(args)











import argparse
from pathlib import Path
from typing import Dict, OrderedDict, Tuple

import torch
from audiocraft.models import MusicGen

from transformers import (
    AutoFeatureExtractor,
    AutoTokenizer,
    EncodecModel,
    MusicgenDecoderConfig,
    MusicgenForConditionalGeneration,
    MusicgenProcessor,
    TaEncoderModel,
)
from transformers.models.musicgen.modeling_musicgen import MusicgenForCausalLM
from transformers.utils import logging


logging.set_verbosity_info()
SCREAMING_SNAKE_CASE_   :      Optional[Any]			=   logging.get_logger(__name__)


SCREAMING_SNAKE_CASE_   :      Any			=   ['''model.decoder.embed_positions.weights''']



def  SCREAMING_SNAKE_CASE				(      snake_case							)		->    int:
		if "emb" in name:
				__lowercase											=     name.replace('emb'						,   'model.decoder.embed_tokens'							)
		if "transformer" in name:
				__lowercase											=     name.replace('transformer'						,   'model.decoder'							)
		if "cross_attention" in name:
				__lowercase											=     name.replace('cross_attention'						,   'encoder_attn'							)
		if "linear1" in name:
				__lowercase											=     name.replace('linear1'						,   'fc1'							)
		if "linear2" in name:
				__lowercase											=     name.replace('linear2'						,   'fc2'							)
		if "norm1" in name:
				__lowercase											=     name.replace('norm1'						,   'self_attn_layer_norm'							)
		if "norm_cross" in name:
				__lowercase											=     name.replace('norm_cross'						,   'encoder_attn_layer_norm'							)
		if "norm2" in name:
				__lowercase											=     name.replace('norm2'						,   'final_layer_norm'							)
		if "out_norm" in name:
				__lowercase											=     name.replace('out_norm'						,   'model.decoder.layer_norm'							)
		if "linears" in name:
				__lowercase											=     name.replace('linears'						,   'lm_heads'							)
		if "condition_provider.conditioners.description.output_proj" in name:
				__lowercase											=     name.replace('condition_provider.conditioners.description.output_proj'						,   'enc_to_dec_proj'							)
		return name



def  SCREAMING_SNAKE_CASE				(      snake_case						,   snake_case							)		->    Tuple[Dict, Dict]:
		__lowercase											=     list(state_dict.keys()							)
		__lowercase											=     {}
		for key in keys:
				__lowercase											=     state_dict.pop(snake_case							)
				__lowercase											=     rename_keys(snake_case							)
				if "in_proj_weight" in key:
						# split fused qkv proj
						__lowercase											=     val[:hidden_size, :]
						__lowercase											=     val[hidden_size : 2 * hidden_size, :]
						__lowercase											=     val[-hidden_size:, :]
				elif "enc_to_dec_proj" in key:
						__lowercase											=     val
				else:
						__lowercase											=     val
		return state_dict, enc_dec_proj_state_dict



def  SCREAMING_SNAKE_CASE				(      snake_case							)		->    MusicgenDecoderConfig:
		if checkpoint == "small":
				# default config values
				__lowercase											=     1_024
				__lowercase											=     24
				__lowercase											=     16
		elif checkpoint == "medium":
				__lowercase											=     1_536
				__lowercase											=     48
				__lowercase											=     24
		elif checkpoint == "large":
				__lowercase											=     2_048
				__lowercase											=     48
				__lowercase											=     32
		else:
				raise ValueError(F"Checkpoint should be one of `['small', 'medium', 'large']`, got {checkpoint}."							)
		__lowercase											=     MusicgenDecoderConfig(
		    hidden_size=snake_case						,   ffn_dim=hidden_size * 4						,   num_hidden_layers=snake_case						,   num_attention_heads=snake_case						,   )
		return config



@torch.no_grad()
def  SCREAMING_SNAKE_CASE				(      snake_case						,   snake_case=None						,   snake_case=None						,   snake_case="cpu"							)		->    Optional[int]:
		__lowercase											=     MusicGen.get_pretrained(snake_case						,   device=snake_case							)
		__lowercase											=     decoder_config_from_checkpoint(snake_case							)

		__lowercase											=     fairseq_model.lm.state_dict()
		__lowercase						,      __lowercase											=     rename_state_dict(
		    snake_case						,   hidden_size=decoder_config.hidden_size							)

		__lowercase											=     TaEncoderModel.from_pretrained('t5-base'							)
		__lowercase											=     EncodecModel.from_pretrained('facebook/encodec_32khz'							)
		__lowercase											=     MusicgenForCausalLM(snake_case							).eval()

		# load all decoder weights - expect that we'll be missing embeddings and enc-dec projection
		__lowercase						,      __lowercase											=     decoder.load_state_dict(snake_case						,   strict=snake_case							)

		for key in missing_keys.copy():
				if key.startswith(('text_encoder', 'audio_encoder')							) or key in EXPECTED_MISSING_KEYS:
						missing_keys.remove(snake_case							)

		if len(snake_case							) > 0:
				raise ValueError(F"Missing key(s) in state_dict: {missing_keys}"							)

		if len(snake_case							) > 0:
				raise ValueError(F"Unexpected key(s) in state_dict: {unexpected_keys}"							)

		# init the composite model
		__lowercase											=     MusicgenForConditionalGeneration(text_encoder=snake_case						,   audio_encoder=snake_case						,   decoder=snake_case							)

		# load the pre-trained enc-dec projection (from the decoder state dict)
		model.enc_to_dec_proj.load_state_dict(snake_case							)

		# check we can do a forward pass
		__lowercase											=     torch.arange(0						,   8						,   dtype=torch.long							).reshape(2						,   -1							)
		__lowercase											=     input_ids.reshape(2 * 4						,   -1							)

		with torch.no_grad():
				__lowercase											=     model(input_ids=snake_case						,   decoder_input_ids=snake_case							).logits

		if logits.shape != (8, 1, 2_048):
				raise ValueError('Incorrect shape for logits'							)

		# now construct the processor
		__lowercase											=     AutoTokenizer.from_pretrained('t5-base'							)
		__lowercase											=     AutoFeatureExtractor.from_pretrained('facebook/encodec_32khz'						,   padding_side='left'							)

		__lowercase											=     MusicgenProcessor(feature_extractor=snake_case						,   tokenizer=snake_case							)

		# set the appropriate bos/pad token ids
		__lowercase											=     2_048
		__lowercase											=     2_048

		# set other default generation config params
		__lowercase											=     int(30 * audio_encoder.config.frame_rate							)
		__lowercase											=     True
		__lowercase											=     3.0

		if pytorch_dump_folder is not None:
				Path(snake_case							).mkdir(exist_ok=snake_case							)
				logger.info(F"Saving model {checkpoint} to {pytorch_dump_folder}"							)
				model.save_pretrained(snake_case							)
				processor.save_pretrained(snake_case							)

		if repo_id:
				logger.info(F"Pushing model {checkpoint} to {repo_id}"							)
				model.push_to_hub(snake_case							)
				processor.push_to_hub(snake_case							)


if __name__ == "__main__":
		SCREAMING_SNAKE_CASE_   :      Optional[int]			=   argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '''--checkpoint''',
		    default='''small''',
		    type=str,
		    help='''Checkpoint size of the MusicGen model you\'d like to convert. Can be one of: `[\'small\', \'medium\', \'large\']`.''',
		)
		parser.add_argument(
		    '''--pytorch_dump_folder''',
		    required=True,
		    default=None,
		    type=str,
		    help='''Path to the output PyTorch model directory.''',
		)
		parser.add_argument(
		    '''--push_to_hub''', default=None, type=str, help='''Where to upload the converted model on the 🤗 hub.'''
		)
		parser.add_argument(
		    '''--device''', default='''cpu''', type=str, help='''Torch device to run the conversion, either cpu or cuda.'''
		)

		SCREAMING_SNAKE_CASE_   :      Any			=   parser.parse_args()
		convert_musicgen_checkpoint(args.checkpoint, args.pytorch_dump_folder, args.push_to_hub)




