




'''simple docstring'''


import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case   :   Any     							=	get_tests_dir('fixtures/test_sentencepiece.model')

if is_torch_available():
    from transformers.models.mbart.modeling_mbart import shift_tokens_right

_snake_case   :   int     							=	250004
_snake_case   :   Dict     							=	250020

@require_sentencepiece
@require_tokenizers
class     A					(	_snake_case  ,unittest.TestCase					):
       lowercase_           =       MBartaaTokenizer
       lowercase_           =       MBartaaTokenizerFast
       lowercase_           =       True
       lowercase_           =       True







       def      __lowerCAmelCase    (						self :						int      )    ->							Any:


              """simple docstring"""


              super().setUp()

              # We have a SentencePiece fixture for testing
              _a			  =       MBartaaTokenizer(lowerCAmelCase_							,    src_lang='''en_XX'''							,    tgt_lang='''ro_RO'''							,    keep_accents=lowerCAmelCase_      )
              tokenizer.save_pretrained(self.tmpdirname      )







       def      __lowerCAmelCase    (						self :						Dict      )    ->							Optional[Any]:


              """simple docstring"""


              _a			  =       '''<s>'''
              _a			  =       0

              self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase_      )							,    lowerCAmelCase_      )
              self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase_      )							,    lowerCAmelCase_      )







       def      __lowerCAmelCase    (						self :						Union[str, Any]      )    ->							List[str]:


              """simple docstring"""


              _a			  =       list(self.get_tokenizer().get_vocab().keys()      )

              self.assertEqual(vocab_keys[0]							,    '''<s>'''      )
              self.assertEqual(vocab_keys[1]							,    '''<pad>'''      )
              self.assertEqual(vocab_keys[-1]							,    '''<mask>'''      )
              self.assertEqual(len(lowerCAmelCase_      )							,    10_54      )







       def      __lowerCAmelCase    (						self :						str      )    ->							Optional[Any]:


              """simple docstring"""


              self.assertEqual(self.get_tokenizer().vocab_size							,    10_54      )







       def      __lowerCAmelCase    (						self :						List[Any]      )    ->							str:


              """simple docstring"""


              _a			  =       MBartaaTokenizer(lowerCAmelCase_							,    src_lang='''en_XX'''							,    tgt_lang='''ro_RO'''							,    keep_accents=lowerCAmelCase_      )

              _a			  =       tokenizer.tokenize('''This is a test'''      )
              self.assertListEqual(lowerCAmelCase_							,    ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

              self.assertListEqual(
                  tokenizer.convert_tokens_to_ids(lowerCAmelCase_      )							,    [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]							,    )

              _a			  =       tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
              self.assertListEqual(
                  lowerCAmelCase_							,    [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.''']							,    )
              _a			  =       tokenizer.convert_tokens_to_ids(lowerCAmelCase_      )
              self.assertListEqual(
                  lowerCAmelCase_							,    [
                      value + tokenizer.fairseq_offset
                      for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                  ]							,    )

              _a			  =       tokenizer.convert_ids_to_tokens(lowerCAmelCase_      )
              self.assertListEqual(
                  lowerCAmelCase_							,    [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.''']							,    )







       @slow
       def      __lowerCAmelCase    (						self :						Optional[Any]      )    ->							Dict:


              """simple docstring"""


              _a			  =       {'''input_ids''': [[25_00_04, 1_10_62, 8_27_72, 7, 15, 8_27_72, 5_38, 5_15_29, 2_37, 1_71_98, 12_90, 2_06, 9, 21_51_75, 13_14, 1_36, 1_71_98, 12_90, 2_06, 9, 5_63_59, 42, 12_20_09, 9, 1_64_66, 16, 8_73_44, 45_37, 9, 47_17, 7_83_81, 6, 15_99_58, 7, 15, 2_44_80, 6_18, 4, 5_27, 2_26_93, 54_28, 4, 27_77, 2_44_80, 98_74, 4, 4_35_23, 5_94, 4, 8_03, 1_83_92, 3_31_89, 18, 4, 4_35_23, 2_44_47, 1_23_99, 1_00, 2_49_55, 8_36_58, 96_26, 14_40_57, 15, 8_39, 2_23_35, 16, 1_36, 2_49_55, 8_36_58, 8_34_79, 15, 3_91_02, 7_24, 16, 6_78, 6_45, 27_89, 13_28, 45_89, 42, 12_20_09, 11_57_74, 23, 8_05, 13_28, 4_68_76, 7, 1_36, 5_38_94, 19_40, 4_22_27, 4_11_59, 1_77_21, 8_23, 4_25, 4, 2_75_12, 9_87_22, 2_06, 1_36, 55_31, 49_70, 9_19, 1_73_36, 5, 2], [25_00_04, 2_00_80, 6_18, 83, 8_27_75, 47, 4_79, 9, 15_17, 73, 5_38_94, 3_33, 8_05_81, 11_01_17, 1_88_11, 52_56, 12_95, 51, 15_25_26, 2_97, 79_86, 3_90, 12_44_16, 5_38, 3_54_31, 2_14, 98, 1_50_44, 2_57_37, 1_36, 71_08, 4_37_01, 23, 7_56, 13_53_55, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [25_00_04, 5_81, 6_37_73, 11_94_55, 6, 14_77_97, 8_82_03, 7, 6_45, 70, 21, 32_85, 1_02_69, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
              # fmt: on

              self.tokenizer_integration_test_util(
                  expected_encoding=lowerCAmelCase_							,    model_name='''facebook/mbart-large-50'''							,    revision='''d3913889c59cd5c9e456b269c376325eabad57e2'''							,    )







       def      __lowerCAmelCase    (						self :						Tuple      )    ->							Dict:


              """simple docstring"""


              if not self.test_slow_tokenizer:
                     # as we don't have a slow version, we can't compare the outputs between slow and fast versions
                     return

              _a			  =       (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-mbart50''', {})
              for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                     with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
                            _a			  =       self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_							,    **lowerCAmelCase_      )
                            _a			  =       self.tokenizer_class.from_pretrained(lowerCAmelCase_							,    **lowerCAmelCase_      )

                            _a			  =       tempfile.mkdtemp()

                            _a			  =       tokenizer_r.save_pretrained(lowerCAmelCase_      )
                            _a			  =       tokenizer_p.save_pretrained(lowerCAmelCase_      )

                            # Checks it save with the same files + the tokenizer.json file for the fast one
                            self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files      )      )
                            _a			  =       tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f      )
                            self.assertSequenceEqual(lowerCAmelCase_							,    lowerCAmelCase_      )

                            # Checks everything loads correctly in the same way
                            _a			  =       tokenizer_r.from_pretrained(lowerCAmelCase_      )
                            _a			  =       tokenizer_p.from_pretrained(lowerCAmelCase_      )

                            # Check special tokens are set accordingly on Rust and Python
                            for key in tokenizer_pp.special_tokens_map:
                                   self.assertTrue(hasattr(lowerCAmelCase_							,    lowerCAmelCase_      )      )
                                   # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
                                   # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

                            shutil.rmtree(lowerCAmelCase_      )

                            # Save tokenizer rust, legacy_format=True
                            _a			  =       tempfile.mkdtemp()

                            _a			  =       tokenizer_r.save_pretrained(lowerCAmelCase_							,    legacy_format=lowerCAmelCase_      )
                            _a			  =       tokenizer_p.save_pretrained(lowerCAmelCase_      )

                            # Checks it save with the same files
                            self.assertSequenceEqual(lowerCAmelCase_							,    lowerCAmelCase_      )

                            # Checks everything loads correctly in the same way
                            _a			  =       tokenizer_r.from_pretrained(lowerCAmelCase_      )
                            _a			  =       tokenizer_p.from_pretrained(lowerCAmelCase_      )

                            # Check special tokens are set accordingly on Rust and Python
                            for key in tokenizer_pp.special_tokens_map:
                                   self.assertTrue(hasattr(lowerCAmelCase_							,    lowerCAmelCase_      )      )

                            shutil.rmtree(lowerCAmelCase_      )

                            # Save tokenizer rust, legacy_format=False
                            _a			  =       tempfile.mkdtemp()

                            _a			  =       tokenizer_r.save_pretrained(lowerCAmelCase_							,    legacy_format=lowerCAmelCase_      )
                            _a			  =       tokenizer_p.save_pretrained(lowerCAmelCase_      )

                            # Checks it saved the tokenizer.json file
                            self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files      )      )

                            # Checks everything loads correctly in the same way
                            _a			  =       tokenizer_r.from_pretrained(lowerCAmelCase_      )
                            _a			  =       tokenizer_p.from_pretrained(lowerCAmelCase_      )

                            # Check special tokens are set accordingly on Rust and Python
                            for key in tokenizer_pp.special_tokens_map:
                                   self.assertTrue(hasattr(lowerCAmelCase_							,    lowerCAmelCase_      )      )

                            shutil.rmtree(lowerCAmelCase_      )



@require_torch
@require_sentencepiece
@require_tokenizers
class     A					(	unittest.TestCase					):
       lowercase_           =       'facebook/mbart-large-50-one-to-many-mmt'
       lowercase_           =       [
           ' UN Chief Says There Is No Military Solution in Syria',
           ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
       ]
       lowercase_           =       [
           'Şeful ONU declară că nu există o soluţie militară în Siria',
           'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
           ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
           ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
       ]
       lowercase_           =       [EN_CODE, 8274, 12_7873, 2_5916, 7, 8622, 2071, 438, 6_7485, 53, 18_7895, 23, 5_1712, 2]







       @classmethod
       def      __lowerCAmelCase    (						cls :						List[str]      )    ->							str:


              """simple docstring"""


              _a			  =       MBartaaTokenizer.from_pretrained(
                  cls.checkpoint_name							,    src_lang='''en_XX'''							,    tgt_lang='''ro_RO'''      )
              _a			  =       1
              return cls







       def      __lowerCAmelCase    (						self :						Tuple      )    ->							Optional[int]:


              """simple docstring"""


              self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ar_AR''']							,    25_00_01      )
              self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''en_EN''']							,    25_00_04      )
              self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ro_RO''']							,    25_00_20      )
              self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''mr_IN''']							,    25_00_38      )







       def      __lowerCAmelCase    (						self :						List[str]      )    ->							Optional[int]:


              """simple docstring"""


              _a			  =       self.tokenizer.batch_encode_plus(self.src_text      ).input_ids[0]
              self.assertListEqual(self.expected_src_tokens							,    lowerCAmelCase_      )







       def      __lowerCAmelCase    (						self :						Dict      )    ->							Optional[Any]:


              """simple docstring"""


              self.assertIn(lowerCAmelCase_							,    self.tokenizer.all_special_ids      )
              _a			  =       [RO_CODE, 8_84, 90_19, 96, 9, 9_16, 8_67_92, 36, 1_87_43, 1_55_96, 5, 2]
              _a			  =       self.tokenizer.decode(lowerCAmelCase_							,    skip_special_tokens=lowerCAmelCase_      )
              _a			  =       self.tokenizer.decode(generated_ids[1:]							,    skip_special_tokens=lowerCAmelCase_      )
              self.assertEqual(lowerCAmelCase_							,    lowerCAmelCase_      )
              self.assertNotIn(self.tokenizer.eos_token							,    lowerCAmelCase_      )







       def      __lowerCAmelCase    (						self :						str      )    ->							Optional[Any]:


              """simple docstring"""


              _a			  =       ['''this is gunna be a long sentence ''' * 20]
              assert isinstance(src_text[0]							,    lowerCAmelCase_      )
              _a			  =       10
              _a			  =       self.tokenizer(lowerCAmelCase_							,    max_length=lowerCAmelCase_							,    truncation=lowerCAmelCase_      ).input_ids[0]
              self.assertEqual(ids[0]							,    lowerCAmelCase_      )
              self.assertEqual(ids[-1]							,    2      )
              self.assertEqual(len(lowerCAmelCase_      )							,    lowerCAmelCase_      )







       def      __lowerCAmelCase    (						self :						Union[str, Any]      )    ->							Any:


              """simple docstring"""


              self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']      )							,    [25_00_53, 25_00_01]      )







       def      __lowerCAmelCase    (						self :						Optional[Any]      )    ->							Optional[int]:


              """simple docstring"""


              _a			  =       tempfile.mkdtemp()
              _a			  =       self.tokenizer.fairseq_tokens_to_ids
              self.tokenizer.save_pretrained(lowerCAmelCase_      )
              _a			  =       MBartaaTokenizer.from_pretrained(lowerCAmelCase_      )
              self.assertDictEqual(new_tok.fairseq_tokens_to_ids							,    lowerCAmelCase_      )







       @require_torch
       def      __lowerCAmelCase    (						self :						Optional[int]      )    ->							Dict:


              """simple docstring"""


              _a			  =       self.tokenizer(self.src_text							,    text_target=self.tgt_text							,    padding=lowerCAmelCase_							,    return_tensors='''pt'''      )
              _a			  =       shift_tokens_right(batch['''labels''']							,    self.tokenizer.pad_token_id      )

              # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
              assert batch.input_ids[1][0] == EN_CODE
              assert batch.input_ids[1][-1] == 2
              assert batch.labels[1][0] == RO_CODE
              assert batch.labels[1][-1] == 2
              assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]







       @require_torch
       def      __lowerCAmelCase    (						self :						List[str]      )    ->							Optional[int]:


              """simple docstring"""


              _a			  =       self.tokenizer(
                  self.src_text							,    text_target=self.tgt_text							,    padding=lowerCAmelCase_							,    truncation=lowerCAmelCase_							,    max_length=len(self.expected_src_tokens      )							,    return_tensors='''pt'''							,    )
              _a			  =       shift_tokens_right(batch['''labels''']							,    self.tokenizer.pad_token_id      )

              self.assertIsInstance(lowerCAmelCase_							,    lowerCAmelCase_      )

              self.assertEqual((2, 14)							,    batch.input_ids.shape      )
              self.assertEqual((2, 14)							,    batch.attention_mask.shape      )
              _a			  =       batch.input_ids.tolist()[0]
              self.assertListEqual(self.expected_src_tokens							,    lowerCAmelCase_      )
              self.assertEqual(2							,    batch.decoder_input_ids[0, 0]      )  # decoder_start_token_id
              # Test that special tokens are reset
              self.assertEqual(self.tokenizer.prefix_tokens							,    [EN_CODE]      )
              self.assertEqual(self.tokenizer.suffix_tokens							,    [self.tokenizer.eos_token_id]      )







       def      __lowerCAmelCase    (						self :						List[Any]      )    ->							str:


              """simple docstring"""


              _a			  =       self.tokenizer(self.src_text							,    padding=lowerCAmelCase_							,    truncation=lowerCAmelCase_							,    max_length=3							,    return_tensors='''pt'''      )
              _a			  =       self.tokenizer(
                  text_target=self.tgt_text							,    padding=lowerCAmelCase_							,    truncation=lowerCAmelCase_							,    max_length=10							,    return_tensors='''pt'''      )
              _a			  =       targets['''input_ids''']
              _a			  =       shift_tokens_right(lowerCAmelCase_							,    self.tokenizer.pad_token_id      )

              self.assertEqual(batch.input_ids.shape[1]							,    3      )
              self.assertEqual(batch.decoder_input_ids.shape[1]							,    10      )







       @require_torch
       def      __lowerCAmelCase    (						self :						List[str]      )    ->							Tuple:


              """simple docstring"""


              _a			  =       self.tokenizer._build_translation_inputs(
                  '''A test'''							,    return_tensors='''pt'''							,    src_lang='''en_XX'''							,    tgt_lang='''ar_AR'''      )

              self.assertEqual(
                  nested_simplify(lowerCAmelCase_      )							,    {
                      # en_XX, A, test, EOS
                      '''input_ids''': [[25_00_04, 62, 30_34, 2]],
                      '''attention_mask''': [[1, 1, 1, 1]],
                      # ar_AR
                      '''forced_bos_token_id''': 25_00_01,
                  }							,    )







'''simple docstring'''


import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
				from transformers import LlamaTokenizerFast
except ImportError as e:
				warnings.warn(e)
				warnings.warn(
				    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
				)
				_snake_case   :   int     							=	None
_snake_case   :   Tuple     							=	{
    '7B': 11008,
    '13B': 13824,
    '30B': 17920,
    '65B': 22016,
    '70B': 28672,
}
_snake_case   :   int     							=	{
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}
def 	snake_case_					(UpperCamelCase :						Union[str, Any]				,		UpperCamelCase :						Optional[Any]=1				,		UpperCamelCase :						Optional[Any]=256   ):





							'''simple docstring'''


							return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3   )   ) + multiple_of - 1) // multiple_of)
def 	snake_case_					(UpperCamelCase :						Optional[int]   ):





							'''simple docstring'''


							with open(UpperCamelCase				,		'''r'''   ) as f:
														return json.load(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						str				,		UpperCamelCase :						str   ):





							'''simple docstring'''


							with open(UpperCamelCase				,		'''w'''   ) as f:
														json.dump(UpperCamelCase				,		UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						Optional[Any]				,		UpperCamelCase :						List[str]				,		UpperCamelCase :						Union[str, Any]				,		UpperCamelCase :						str=True   ):





							'''simple docstring'''


							os.makedirs(UpperCamelCase				,		exist_ok=UpperCamelCase   )
							_a			  =       os.path.join(UpperCamelCase				,		'''tmp'''   )
							os.makedirs(UpperCamelCase				,		exist_ok=UpperCamelCase   )

							_a			  =       read_json(os.path.join(UpperCamelCase				,		'''params.json'''   )   )
							_a			  =       NUM_SHARDS[model_size]
							_a			  =       params['''n_layers''']
							_a			  =       params['''n_heads''']
							_a			  =       n_heads // num_shards
							_a			  =       params['''dim''']
							_a			  =       dim // n_heads
							_a			  =       10000.0
							_a			  =       1.0 / (base ** (torch.arange(0				,		UpperCamelCase				,		2   ).float() / dims_per_head))

							if "n_kv_heads" in params:
														_a			  =       params['''n_kv_heads''']  # for GQA / MQA
														_a			  =       n_heads_per_shard // num_key_value_heads
														_a			  =       dim // num_key_value_heads
							else:  # compatibility with other checkpoints
														_a			  =       n_heads
														_a			  =       n_heads_per_shard
														_a			  =       dim

							# permute for sliced rotary
							def permute(UpperCamelCase :						List[str]				,		UpperCamelCase :						str=n_heads				,		UpperCamelCase :						int=dim				,		UpperCamelCase :						Optional[Any]=dim   ):
														return w.view(UpperCamelCase				,		dima // n_heads // 2				,		2				,		UpperCamelCase   ).transpose(1				,		2   ).reshape(UpperCamelCase				,		UpperCamelCase   )

							print(f'Fetching all parameters from the checkpoint at {input_base_path}.'   )
							# Load weights
							if model_size == "7B":
														# Not sharded
														# (The sharded implementation would also work, but this is simpler.)
														_a			  =       torch.load(os.path.join(UpperCamelCase				,		'''consolidated.00.pth'''   )				,		map_location='''cpu'''   )
							else:
														# Sharded
														_a			  =       [
														    torch.load(os.path.join(UpperCamelCase				,		f'consolidated.{i:02d}.pth'   )				,		map_location='''cpu'''   )
														    for i in range(UpperCamelCase   )
														]
							_a			  =       0
							_a			  =       {'''weight_map''': {}}
							for layer_i in range(UpperCamelCase   ):
														_a			  =       f'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
														if model_size == "7B":
																					# Unsharded
																					_a			  =       {
																					    f'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
																					        loaded[f'layers.{layer_i}.attention.wq.weight']   ),
																					    f'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
																					        loaded[f'layers.{layer_i}.attention.wk.weight']   ),
																					    f'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[f'layers.{layer_i}.attention.wv.weight'],
																					    f'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[f'layers.{layer_i}.attention.wo.weight'],
																					    f'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w1.weight'],
																					    f'model.layers.{layer_i}.mlp.down_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w2.weight'],
																					    f'model.layers.{layer_i}.mlp.up_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w3.weight'],
																					    f'model.layers.{layer_i}.input_layernorm.weight': loaded[f'layers.{layer_i}.attention_norm.weight'],
																					    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[f'layers.{layer_i}.ffn_norm.weight'],
																					}
														else:
																					# Sharded
																					# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																					# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																					# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																					_a			  =       {
																					    f'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
																					        f'layers.{layer_i}.attention_norm.weight'
																					    ].clone(),
																					    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
																					        f'layers.{layer_i}.ffn_norm.weight'
																					    ].clone(),
																					}
																					_a			  =       permute(
																					    torch.cat(
																					        [
																					            loaded[i][f'layers.{layer_i}.attention.wq.weight'].view(UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   )
																					            for i in range(UpperCamelCase   )
																					        ]				,		dim=0				,		).reshape(UpperCamelCase				,		UpperCamelCase   )   )
																					_a			  =       permute(
																					    torch.cat(
																					        [
																					            loaded[i][f'layers.{layer_i}.attention.wk.weight'].view(
																					                UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   )
																					            for i in range(UpperCamelCase   )
																					        ]				,		dim=0				,		).reshape(UpperCamelCase				,		UpperCamelCase   )				,		UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase				,		)
																					_a			  =       torch.cat(
																					    [
																					        loaded[i][f'layers.{layer_i}.attention.wv.weight'].view(
																					            UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   )
																					        for i in range(UpperCamelCase   )
																					    ]				,		dim=0				,		).reshape(UpperCamelCase				,		UpperCamelCase   )

																					_a			  =       torch.cat(
																					    [loaded[i][f'layers.{layer_i}.attention.wo.weight'] for i in range(UpperCamelCase   )]				,		dim=1   )
																					_a			  =       torch.cat(
																					    [loaded[i][f'layers.{layer_i}.feed_forward.w1.weight'] for i in range(UpperCamelCase   )]				,		dim=0   )
																					_a			  =       torch.cat(
																					    [loaded[i][f'layers.{layer_i}.feed_forward.w2.weight'] for i in range(UpperCamelCase   )]				,		dim=1   )
																					_a			  =       torch.cat(
																					    [loaded[i][f'layers.{layer_i}.feed_forward.w3.weight'] for i in range(UpperCamelCase   )]				,		dim=0   )

														_a			  =       inv_freq
														for k, v in state_dict.items():
																					_a			  =       filename
																					param_count += v.numel()
														torch.save(UpperCamelCase				,		os.path.join(UpperCamelCase				,		UpperCamelCase   )   )

							_a			  =       f'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
							if model_size == "7B":
														# Unsharded
														_a			  =       {
														    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
														    '''model.norm.weight''': loaded['''norm.weight'''],
														    '''lm_head.weight''': loaded['''output.weight'''],
														}
							else:
														_a			  =       {
														    '''model.norm.weight''': loaded[0]['''norm.weight'''],
														    '''model.embed_tokens.weight''': torch.cat(
														        [loaded[i]['''tok_embeddings.weight'''] for i in range(UpperCamelCase   )]				,		dim=1   ),
														    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(UpperCamelCase   )]				,		dim=0   ),
														}

							for k, v in state_dict.items():
														_a			  =       filename
														param_count += v.numel()
							torch.save(UpperCamelCase				,		os.path.join(UpperCamelCase				,		UpperCamelCase   )   )

							# Write configs
							_a			  =       {'''total_size''': param_count * 2}
							write_json(UpperCamelCase				,		os.path.join(UpperCamelCase				,		'''pytorch_model.bin.index.json'''   )   )
							_a			  =       params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
							_a			  =       params['''multiple_of'''] if '''multiple_of''' in params else 256
							_a			  =       LlamaConfig(
							    hidden_size=UpperCamelCase				,		intermediate_size=compute_intermediate_size(UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   )				,		num_attention_heads=params['''n_heads''']				,		num_hidden_layers=params['''n_layers''']				,		rms_norm_eps=params['''norm_eps''']				,		num_key_value_heads=UpperCamelCase				,		)
							config.save_pretrained(UpperCamelCase   )

							# Make space so we can load the model properly now.
							del state_dict
							del loaded
							gc.collect()

							print('''Loading the checkpoint in a Llama model.'''   )
							_a			  =       LlamaForCausalLM.from_pretrained(UpperCamelCase				,		torch_dtype=torch.floataa				,		low_cpu_mem_usage=UpperCamelCase   )
							# Avoid saving this as part of the config.
							del model.config._name_or_path

							print('''Saving in the Transformers format.'''   )
							model.save_pretrained(UpperCamelCase				,		safe_serialization=UpperCamelCase   )
							shutil.rmtree(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						Optional[int]				,		UpperCamelCase :						Tuple   ):





							'''simple docstring'''


							_a			  =       LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
							print(f'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'   )
							_a			  =       tokenizer_class(UpperCamelCase   )
							tokenizer.save_pretrained(UpperCamelCase   )
def 	snake_case_					():





							'''simple docstring'''


							_a			  =       argparse.ArgumentParser()
							parser.add_argument(
							    '''--input_dir'''				,		help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''				,		)
							parser.add_argument(
							    '''--model_size'''				,		choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']				,		)
							parser.add_argument(
							    '''--output_dir'''				,		help='''Location to write HF model and tokenizer'''				,		)
							parser.add_argument('''--safe_serialization'''				,		type=UpperCamelCase				,		help='''Whether or not to save using `safetensors`.'''   )
							_a			  =       parser.parse_args()
							if args.model_size != "tokenizer_only":
														write_model(
														    model_path=args.output_dir				,		input_base_path=os.path.join(args.input_dir				,		args.model_size   )				,		model_size=args.model_size				,		safe_serialization=args.safe_serialization				,		)
							_a			  =       os.path.join(args.input_dir				,		'''tokenizer.model'''   )
							write_tokenizer(args.output_dir				,		UpperCamelCase   )


if __name__ == "__main__":
				main()

