




'''simple docstring'''


import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil

class     A					(	_a					):







							def      __lowerCAmelCase    (						self :						List[Any]      )    ->							List[Any]:


														"""simple docstring"""


														_a			  =       pa.array(TypedSequence([1, 2, 3]      )      )
														self.assertEqual(arr.type							,    pa.intaa()      )







							def      __lowerCAmelCase    (						self :						List[str]      )    ->							str:


														"""simple docstring"""


														with self.assertRaises(lowerCAmelCase_      ):
																					_a			  =       pa.array(TypedSequence([1, 2, 3]      )							,    type=pa.intaa()      )







							def      __lowerCAmelCase    (						self :						Dict      )    ->							Dict:


														"""simple docstring"""


														with self.assertRaises(lowerCAmelCase_      ):
																					_a			  =       pa.array(TypedSequence([1, 2, 3]							,    try_type=Value('''bool'''      )							,    type=Value('''int64'''      )      )      )







							def      __lowerCAmelCase    (						self :						List[str]      )    ->							Optional[Any]:


														"""simple docstring"""


														_a			  =       pa.array(TypedSequence([1, 2, 3]							,    type=Value('''int32'''      )      )      )
														self.assertEqual(arr.type							,    pa.intaa()      )







							def      __lowerCAmelCase    (						self :						Any      )    ->							List[Any]:


														"""simple docstring"""


														with self.assertRaises((TypeError, pa.lib.ArrowInvalid)      ):
																					_a			  =       pa.array(TypedSequence(['''foo''', '''bar''']							,    type=Value('''int64'''      )      )      )







							def      __lowerCAmelCase    (						self :						Tuple      )    ->							List[Any]:


														"""simple docstring"""


														_a			  =       pa.array(TypedSequence([1, 2, 3]							,    try_type=Value('''int32'''      )      )      )
														self.assertEqual(arr.type							,    pa.intaa()      )







							def      __lowerCAmelCase    (						self :						Any      )    ->							Tuple:


														"""simple docstring"""


														_a			  =       pa.array(TypedSequence(['''foo''', '''bar''']							,    try_type=Value('''int64'''      )      )      )
														self.assertEqual(arr.type							,    pa.string()      )







							def      __lowerCAmelCase    (						self :						List[Any]      )    ->							str:


														"""simple docstring"""


														_a			  =       pa.array(TypedSequence([[[1, 2, 3]]]							,    type=ArrayaD((1, 3)							,    '''int64'''      )      )      )
														self.assertEqual(arr.type							,    ArrayaDExtensionType((1, 3)							,    '''int64'''      )      )







							def      __lowerCAmelCase    (						self :						Optional[Any]      )    ->							Optional[int]:


														"""simple docstring"""


														with self.assertRaises((TypeError, pa.lib.ArrowInvalid)      ):
																					_a			  =       pa.array(TypedSequence(['''foo''', '''bar''']							,    type=ArrayaD((1, 3)							,    '''int64'''      )      )      )







							def      __lowerCAmelCase    (						self :						Tuple      )    ->							Tuple:


														"""simple docstring"""


														_a			  =       pa.array(TypedSequence([[[1, 2, 3]]]							,    try_type=ArrayaD((1, 3)							,    '''int64'''      )      )      )
														self.assertEqual(arr.type							,    ArrayaDExtensionType((1, 3)							,    '''int64'''      )      )







							def      __lowerCAmelCase    (						self :						List[str]      )    ->							Any:


														"""simple docstring"""


														_a			  =       pa.array(TypedSequence(['''foo''', '''bar''']							,    try_type=ArrayaD((1, 3)							,    '''int64'''      )      )      )
														self.assertEqual(arr.type							,    pa.string()      )







							@require_pil
							def      __lowerCAmelCase    (						self :						Tuple      )    ->							List[Any]:


														"""simple docstring"""


														import PIL.Image

														_a			  =       PIL.Image.fromarray(np.arange(10							,    dtype=np.uinta      ).reshape(2							,    5      )      )
														with patch(
														    '''datasets.arrow_writer.cast_to_python_objects'''							,    side_effect=lowerCAmelCase_      ) as mock_cast_to_python_objects:
																					_a			  =       pa.array(TypedSequence([{'''path''': None, '''bytes''': B'''image_bytes'''}, pil_image]							,    type=Image()      )      )
																					_a  ,							_a			  =       mock_cast_to_python_objects.call_args_list[-1]
																					self.assertIn('''optimize_list_casting'''							,    lowerCAmelCase_      )
																					self.assertFalse(kwargs['''optimize_list_casting''']      )



def 	snake_case_					(UpperCamelCase :						Union[str, Any]				,		UpperCamelCase :						int   ):





							'''simple docstring'''


							_a			  =       pa.BufferReader(UpperCamelCase   ) if isinstance(UpperCamelCase				,		pa.Buffer   ) else pa.memory_map(UpperCamelCase   )
							_a			  =       pa.ipc.open_stream(UpperCamelCase   )
							_a			  =       f.read_all()
							assert len(pa_table.to_batches()   ) == expected_num_chunks
							assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
							del pa_table
@pytest.mark.parametrize('''writer_batch_size'''				,		[None, 1, 10]   )
@pytest.mark.parametrize(
    '''fields'''				,		[None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}]   )
def 	snake_case_					(UpperCamelCase :						Dict				,		UpperCamelCase :						str   ):





							'''simple docstring'''


							_a			  =       pa.BufferOutputStream()
							_a			  =       pa.schema(UpperCamelCase   ) if fields else None
							with ArrowWriter(stream=UpperCamelCase				,		schema=UpperCamelCase				,		writer_batch_size=UpperCamelCase   ) as writer:
														writer.write({'''col_1''': '''foo''', '''col_2''': 1}   )
														writer.write({'''col_1''': '''bar''', '''col_2''': 2}   )
														_a  ,							_a			  =       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							if not fields:
														_a			  =       {'''col_1''': pa.string(), '''col_2''': pa.intaa()}
							assert writer._schema == pa.schema(UpperCamelCase				,		metadata=writer._schema.metadata   )
							_check_output(output.getvalue()				,		expected_num_chunks=num_examples if writer_batch_size == 1 else 1   )
def 	snake_case_					():





							'''simple docstring'''


							_a			  =       pa.BufferOutputStream()
							_a			  =       Features({'''labels''': ClassLabel(names=['''neg''', '''pos''']   )}   )
							with ArrowWriter(stream=UpperCamelCase				,		features=UpperCamelCase   ) as writer:
														writer.write({'''labels''': 0}   )
														writer.write({'''labels''': 1}   )
														_a  ,							_a			  =       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							assert writer._schema == features.arrow_schema
							assert writer._schema.metadata == features.arrow_schema.metadata
							_a			  =       pa.BufferReader(output.getvalue()   )
							_a			  =       pa.ipc.open_stream(UpperCamelCase   )
							_a			  =       f.read_all()
							_a			  =       pa_table.schema
							assert pa_table.num_rows == 2
							assert schema == features.arrow_schema
							assert schema.metadata == features.arrow_schema.metadata
							assert features == Features.from_arrow_schema(UpperCamelCase   )
@pytest.mark.parametrize('''writer_batch_size'''				,		[None, 1, 10]   )
def 	snake_case_					(UpperCamelCase :						Any   ):





							'''simple docstring'''


							_a			  =       pa.BufferOutputStream()
							with ArrowWriter(
							    stream=UpperCamelCase				,		writer_batch_size=UpperCamelCase				,		hash_salt='''split_name'''				,		check_duplicates=UpperCamelCase				,		) as writer:
														with pytest.raises(UpperCamelCase   ):
																					writer.write({'''col_1''': '''foo''', '''col_2''': 1}				,		key=[1, 2]   )
																					_a  ,							_a			  =       writer.finalize()
@pytest.mark.parametrize('''writer_batch_size'''				,		[None, 2, 10]   )
def 	snake_case_					(UpperCamelCase :						Any   ):





							'''simple docstring'''


							_a			  =       pa.BufferOutputStream()
							with ArrowWriter(
							    stream=UpperCamelCase				,		writer_batch_size=UpperCamelCase				,		hash_salt='''split_name'''				,		check_duplicates=UpperCamelCase				,		) as writer:
														with pytest.raises(UpperCamelCase   ):
																					writer.write({'''col_1''': '''foo''', '''col_2''': 1}				,		key=10   )
																					writer.write({'''col_1''': '''bar''', '''col_2''': 2}				,		key=10   )
																					_a  ,							_a			  =       writer.finalize()
@pytest.mark.parametrize('''writer_batch_size'''				,		[None, 2, 10]   )
def 	snake_case_					(UpperCamelCase :						int   ):





							'''simple docstring'''


							_a			  =       pa.BufferOutputStream()
							with ArrowWriter(
							    stream=UpperCamelCase				,		writer_batch_size=UpperCamelCase				,		hash_salt='''split_name'''				,		check_duplicates=UpperCamelCase				,		) as writer:
														writer.write({'''col_1''': '''foo''', '''col_2''': 1}				,		key=1   )
														writer.write({'''col_1''': '''bar''', '''col_2''': 2}				,		key=2   )
														_a  ,							_a			  =       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							_check_output(output.getvalue()				,		expected_num_chunks=num_examples if writer_batch_size == 1 else 1   )
@pytest.mark.parametrize('''writer_batch_size'''				,		[None, 1, 10]   )
@pytest.mark.parametrize(
    '''fields'''				,		[None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}]   )
def 	snake_case_					(UpperCamelCase :						Dict				,		UpperCamelCase :						List[str]   ):





							'''simple docstring'''


							_a			  =       pa.BufferOutputStream()
							_a			  =       pa.schema(UpperCamelCase   ) if fields else None
							with ArrowWriter(stream=UpperCamelCase				,		schema=UpperCamelCase				,		writer_batch_size=UpperCamelCase   ) as writer:
														writer.write_batch({'''col_1''': ['''foo''', '''bar'''], '''col_2''': [1, 2]}   )
														writer.write_batch({'''col_1''': [], '''col_2''': []}   )
														_a  ,							_a			  =       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							if not fields:
														_a			  =       {'''col_1''': pa.string(), '''col_2''': pa.intaa()}
							assert writer._schema == pa.schema(UpperCamelCase				,		metadata=writer._schema.metadata   )
							_check_output(output.getvalue()				,		expected_num_chunks=num_examples if writer_batch_size == 1 else 1   )
@pytest.mark.parametrize('''writer_batch_size'''				,		[None, 1, 10]   )
@pytest.mark.parametrize(
    '''fields'''				,		[None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}]   )
def 	snake_case_					(UpperCamelCase :						Any				,		UpperCamelCase :						Any   ):





							'''simple docstring'''


							_a			  =       pa.BufferOutputStream()
							_a			  =       pa.schema(UpperCamelCase   ) if fields else None
							with ArrowWriter(stream=UpperCamelCase				,		schema=UpperCamelCase				,		writer_batch_size=UpperCamelCase   ) as writer:
														writer.write_table(pa.Table.from_pydict({'''col_1''': ['''foo''', '''bar'''], '''col_2''': [1, 2]}   )   )
														_a  ,							_a			  =       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							if not fields:
														_a			  =       {'''col_1''': pa.string(), '''col_2''': pa.intaa()}
							assert writer._schema == pa.schema(UpperCamelCase				,		metadata=writer._schema.metadata   )
							_check_output(output.getvalue()				,		expected_num_chunks=num_examples if writer_batch_size == 1 else 1   )
@pytest.mark.parametrize('''writer_batch_size'''				,		[None, 1, 10]   )
@pytest.mark.parametrize(
    '''fields'''				,		[None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}]   )
def 	snake_case_					(UpperCamelCase :						int				,		UpperCamelCase :						Optional[int]   ):





							'''simple docstring'''


							_a			  =       pa.BufferOutputStream()
							_a			  =       pa.schema(UpperCamelCase   ) if fields else None
							with ArrowWriter(stream=UpperCamelCase				,		schema=UpperCamelCase				,		writer_batch_size=UpperCamelCase   ) as writer:
														writer.write_row(pa.Table.from_pydict({'''col_1''': ['''foo'''], '''col_2''': [1]}   )   )
														writer.write_row(pa.Table.from_pydict({'''col_1''': ['''bar'''], '''col_2''': [2]}   )   )
														_a  ,							_a			  =       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							if not fields:
														_a			  =       {'''col_1''': pa.string(), '''col_2''': pa.intaa()}
							assert writer._schema == pa.schema(UpperCamelCase				,		metadata=writer._schema.metadata   )
							_check_output(output.getvalue()				,		expected_num_chunks=num_examples if writer_batch_size == 1 else 1   )
def 	snake_case_					():





							'''simple docstring'''


							with tempfile.TemporaryDirectory() as tmp_dir:
														_a			  =       {'''col_1''': pa.string(), '''col_2''': pa.intaa()}
														_a			  =       os.path.join(UpperCamelCase				,		'''test.arrow'''   )
														with ArrowWriter(path=UpperCamelCase				,		schema=pa.schema(UpperCamelCase   )   ) as writer:
																					writer.write_batch({'''col_1''': ['''foo''', '''bar'''], '''col_2''': [1, 2]}   )
																					_a  ,							_a			  =       writer.finalize()
														assert num_examples == 2
														assert num_bytes > 0
														assert writer._schema == pa.schema(UpperCamelCase				,		metadata=writer._schema.metadata   )
														_check_output(UpperCamelCase				,		1   )
def 	snake_case_					(UpperCamelCase :						List[str]   ):





							'''simple docstring'''


							if pa.types.is_list(UpperCamelCase   ):
														return get_base_dtype(arr_type.value_type   )
							else:
														return arr_type
def 	snake_case_					(UpperCamelCase :						Optional[int]				,		UpperCamelCase :						Any   ):





							'''simple docstring'''


							if isinstance(lst[0]				,		UpperCamelCase   ):
														change_first_primitive_element_in_list(lst[0]				,		UpperCamelCase   )
							else:
														_a			  =       value
@pytest.mark.parametrize('''optimized_int_type, expected_dtype'''				,		[(None, pa.intaa()), (Value('''int32'''   ), pa.intaa())]   )
@pytest.mark.parametrize('''sequence'''				,		[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]   )
def 	snake_case_					(UpperCamelCase :						int				,		UpperCamelCase :						Tuple				,		UpperCamelCase :						Optional[int]   ):





							'''simple docstring'''


							_a			  =       pa.array(TypedSequence(UpperCamelCase				,		optimized_int_type=UpperCamelCase   )   )
							assert get_base_dtype(arr.type   ) == expected_dtype
@pytest.mark.parametrize(
    '''col, expected_dtype'''				,		[
        ('''attention_mask''', pa.inta()),
        ('''special_tokens_mask''', pa.inta()),
        ('''token_type_ids''', pa.inta()),
        ('''input_ids''', pa.intaa()),
        ('''other''', pa.intaa()),
    ]				,		)
@pytest.mark.parametrize('''sequence'''				,		[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]   )
def 	snake_case_					(UpperCamelCase :						List[Any]				,		UpperCamelCase :						str				,		UpperCamelCase :						List[Any]   ):





							'''simple docstring'''


							_a			  =       pa.array(OptimizedTypedSequence(UpperCamelCase				,		col=UpperCamelCase   )   )
							assert get_base_dtype(arr.type   ) == expected_dtype

							# not in range
							if col != "other":
														# avoids errors due to in-place modifications
														_a			  =       copy.deepcopy(UpperCamelCase   )
														_a			  =       np.iinfo(expected_dtype.to_pandas_dtype()   ).max + 1
														change_first_primitive_element_in_list(UpperCamelCase				,		UpperCamelCase   )
														_a			  =       pa.array(OptimizedTypedSequence(UpperCamelCase				,		col=UpperCamelCase   )   )
														assert get_base_dtype(arr.type   ) == pa.intaa()
@pytest.mark.parametrize('''raise_exception'''				,		[False, True]   )
def 	snake_case_					(UpperCamelCase :						Optional[Any]				,		UpperCamelCase :						Optional[Any]   ):





							'''simple docstring'''


							_a			  =       str(tmp_path / '''dataset-train.arrow'''   )
							try:
														with ArrowWriter(path=UpperCamelCase   ) as writer:
																					if raise_exception:
																												raise pa.lib.ArrowInvalid()
																					else:
																												writer.stream.close()
							except pa.lib.ArrowInvalid:
														pass
							finally:
														assert writer.stream.closed
def 	snake_case_					(UpperCamelCase :						List[Any]   ):





							'''simple docstring'''


							_a			  =       '''mock://dataset-train.arrow'''
							with ArrowWriter(path=UpperCamelCase				,		storage_options=mockfs.storage_options   ) as writer:
														assert isinstance(writer._fs				,		type(UpperCamelCase   )   )
														assert writer._fs.storage_options == mockfs.storage_options
														writer.write({'''col_1''': '''foo''', '''col_2''': 1}   )
														writer.write({'''col_1''': '''bar''', '''col_2''': 2}   )
														_a  ,							_a			  =       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							assert mockfs.exists(UpperCamelCase   )
def 	snake_case_					():





							'''simple docstring'''


							_a			  =       pa.BufferOutputStream()
							with ParquetWriter(stream=UpperCamelCase   ) as writer:
														writer.write({'''col_1''': '''foo''', '''col_2''': 1}   )
														writer.write({'''col_1''': '''bar''', '''col_2''': 2}   )
														_a  ,							_a			  =       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							_a			  =       pa.BufferReader(output.getvalue()   )
							_a			  =       pq.read_table(UpperCamelCase   )
							assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
@require_pil
@pytest.mark.parametrize('''embed_local_files'''				,		[False, True]   )
def 	snake_case_					(UpperCamelCase :						List[Any]				,		UpperCamelCase :						List[Any]   ):





							'''simple docstring'''


							import PIL.Image

							_a			  =       str(tmp_path / '''test_image_rgb.jpg'''   )
							PIL.Image.fromarray(np.zeros((5, 5)				,		dtype=np.uinta   )   ).save(UpperCamelCase				,		format='''png'''   )
							_a			  =       pa.BufferOutputStream()
							with ParquetWriter(
							    stream=UpperCamelCase				,		features=Features({'''image''': Image()}   )				,		embed_local_files=UpperCamelCase   ) as writer:
														writer.write({'''image''': image_path}   )
														writer.finalize()
							_a			  =       pa.BufferReader(output.getvalue()   )
							_a			  =       pq.read_table(UpperCamelCase   )
							_a			  =       pa_table.to_pydict()
							if embed_local_files:
														assert isinstance(out['''image'''][0]['''path''']				,		UpperCamelCase   )
														with open(UpperCamelCase				,		'''rb'''   ) as f:
																					assert out["image"][0]["bytes"] == f.read()
							else:
														assert out["image"][0]["path"] == image_path
														assert out["image"][0]["bytes"] is None

def 	snake_case_					():





							'''simple docstring'''


							_a			  =       pa.schema([pa.field('''col_1'''				,		pa.string()				,		nullable=UpperCamelCase   )]   )
							_a			  =       pa.BufferOutputStream()
							with ArrowWriter(stream=UpperCamelCase   ) as writer:
														writer._build_writer(inferred_schema=UpperCamelCase   )
														assert writer._schema == pa.schema([pa.field('''col_1'''				,		pa.string()   )]   )







'''simple docstring'''


import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
				from transformers import LlamaTokenizerFast
except ImportError as e:
				warnings.warn(e)
				warnings.warn(
				    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
				)
				_snake_case   :   int     							=	None
_snake_case   :   Tuple     							=	{
    '7B': 11008,
    '13B': 13824,
    '30B': 17920,
    '65B': 22016,
    '70B': 28672,
}
_snake_case   :   int     							=	{
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}
def 	snake_case_					(UpperCamelCase :						Union[str, Any]				,		UpperCamelCase :						Optional[Any]=1				,		UpperCamelCase :						Optional[Any]=256   ):





							'''simple docstring'''


							return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3   )   ) + multiple_of - 1) // multiple_of)
def 	snake_case_					(UpperCamelCase :						Optional[int]   ):





							'''simple docstring'''


							with open(UpperCamelCase				,		'''r'''   ) as f:
														return json.load(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						str				,		UpperCamelCase :						str   ):





							'''simple docstring'''


							with open(UpperCamelCase				,		'''w'''   ) as f:
														json.dump(UpperCamelCase				,		UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						Optional[Any]				,		UpperCamelCase :						List[str]				,		UpperCamelCase :						Union[str, Any]				,		UpperCamelCase :						str=True   ):





							'''simple docstring'''


							os.makedirs(UpperCamelCase				,		exist_ok=UpperCamelCase   )
							_a			  =       os.path.join(UpperCamelCase				,		'''tmp'''   )
							os.makedirs(UpperCamelCase				,		exist_ok=UpperCamelCase   )

							_a			  =       read_json(os.path.join(UpperCamelCase				,		'''params.json'''   )   )
							_a			  =       NUM_SHARDS[model_size]
							_a			  =       params['''n_layers''']
							_a			  =       params['''n_heads''']
							_a			  =       n_heads // num_shards
							_a			  =       params['''dim''']
							_a			  =       dim // n_heads
							_a			  =       10000.0
							_a			  =       1.0 / (base ** (torch.arange(0				,		UpperCamelCase				,		2   ).float() / dims_per_head))

							if "n_kv_heads" in params:
														_a			  =       params['''n_kv_heads''']  # for GQA / MQA
														_a			  =       n_heads_per_shard // num_key_value_heads
														_a			  =       dim // num_key_value_heads
							else:  # compatibility with other checkpoints
														_a			  =       n_heads
														_a			  =       n_heads_per_shard
														_a			  =       dim

							# permute for sliced rotary
							def permute(UpperCamelCase :						List[str]				,		UpperCamelCase :						str=n_heads				,		UpperCamelCase :						int=dim				,		UpperCamelCase :						Optional[Any]=dim   ):
														return w.view(UpperCamelCase				,		dima // n_heads // 2				,		2				,		UpperCamelCase   ).transpose(1				,		2   ).reshape(UpperCamelCase				,		UpperCamelCase   )

							print(f'Fetching all parameters from the checkpoint at {input_base_path}.'   )
							# Load weights
							if model_size == "7B":
														# Not sharded
														# (The sharded implementation would also work, but this is simpler.)
														_a			  =       torch.load(os.path.join(UpperCamelCase				,		'''consolidated.00.pth'''   )				,		map_location='''cpu'''   )
							else:
														# Sharded
														_a			  =       [
														    torch.load(os.path.join(UpperCamelCase				,		f'consolidated.{i:02d}.pth'   )				,		map_location='''cpu'''   )
														    for i in range(UpperCamelCase   )
														]
							_a			  =       0
							_a			  =       {'''weight_map''': {}}
							for layer_i in range(UpperCamelCase   ):
														_a			  =       f'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
														if model_size == "7B":
																					# Unsharded
																					_a			  =       {
																					    f'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
																					        loaded[f'layers.{layer_i}.attention.wq.weight']   ),
																					    f'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
																					        loaded[f'layers.{layer_i}.attention.wk.weight']   ),
																					    f'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[f'layers.{layer_i}.attention.wv.weight'],
																					    f'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[f'layers.{layer_i}.attention.wo.weight'],
																					    f'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w1.weight'],
																					    f'model.layers.{layer_i}.mlp.down_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w2.weight'],
																					    f'model.layers.{layer_i}.mlp.up_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w3.weight'],
																					    f'model.layers.{layer_i}.input_layernorm.weight': loaded[f'layers.{layer_i}.attention_norm.weight'],
																					    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[f'layers.{layer_i}.ffn_norm.weight'],
																					}
														else:
																					# Sharded
																					# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																					# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																					# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																					_a			  =       {
																					    f'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
																					        f'layers.{layer_i}.attention_norm.weight'
																					    ].clone(),
																					    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
																					        f'layers.{layer_i}.ffn_norm.weight'
																					    ].clone(),
																					}
																					_a			  =       permute(
																					    torch.cat(
																					        [
																					            loaded[i][f'layers.{layer_i}.attention.wq.weight'].view(UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   )
																					            for i in range(UpperCamelCase   )
																					        ]				,		dim=0				,		).reshape(UpperCamelCase				,		UpperCamelCase   )   )
																					_a			  =       permute(
																					    torch.cat(
																					        [
																					            loaded[i][f'layers.{layer_i}.attention.wk.weight'].view(
																					                UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   )
																					            for i in range(UpperCamelCase   )
																					        ]				,		dim=0				,		).reshape(UpperCamelCase				,		UpperCamelCase   )				,		UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase				,		)
																					_a			  =       torch.cat(
																					    [
																					        loaded[i][f'layers.{layer_i}.attention.wv.weight'].view(
																					            UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   )
																					        for i in range(UpperCamelCase   )
																					    ]				,		dim=0				,		).reshape(UpperCamelCase				,		UpperCamelCase   )

																					_a			  =       torch.cat(
																					    [loaded[i][f'layers.{layer_i}.attention.wo.weight'] for i in range(UpperCamelCase   )]				,		dim=1   )
																					_a			  =       torch.cat(
																					    [loaded[i][f'layers.{layer_i}.feed_forward.w1.weight'] for i in range(UpperCamelCase   )]				,		dim=0   )
																					_a			  =       torch.cat(
																					    [loaded[i][f'layers.{layer_i}.feed_forward.w2.weight'] for i in range(UpperCamelCase   )]				,		dim=1   )
																					_a			  =       torch.cat(
																					    [loaded[i][f'layers.{layer_i}.feed_forward.w3.weight'] for i in range(UpperCamelCase   )]				,		dim=0   )

														_a			  =       inv_freq
														for k, v in state_dict.items():
																					_a			  =       filename
																					param_count += v.numel()
														torch.save(UpperCamelCase				,		os.path.join(UpperCamelCase				,		UpperCamelCase   )   )

							_a			  =       f'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
							if model_size == "7B":
														# Unsharded
														_a			  =       {
														    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
														    '''model.norm.weight''': loaded['''norm.weight'''],
														    '''lm_head.weight''': loaded['''output.weight'''],
														}
							else:
														_a			  =       {
														    '''model.norm.weight''': loaded[0]['''norm.weight'''],
														    '''model.embed_tokens.weight''': torch.cat(
														        [loaded[i]['''tok_embeddings.weight'''] for i in range(UpperCamelCase   )]				,		dim=1   ),
														    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(UpperCamelCase   )]				,		dim=0   ),
														}

							for k, v in state_dict.items():
														_a			  =       filename
														param_count += v.numel()
							torch.save(UpperCamelCase				,		os.path.join(UpperCamelCase				,		UpperCamelCase   )   )

							# Write configs
							_a			  =       {'''total_size''': param_count * 2}
							write_json(UpperCamelCase				,		os.path.join(UpperCamelCase				,		'''pytorch_model.bin.index.json'''   )   )
							_a			  =       params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
							_a			  =       params['''multiple_of'''] if '''multiple_of''' in params else 256
							_a			  =       LlamaConfig(
							    hidden_size=UpperCamelCase				,		intermediate_size=compute_intermediate_size(UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   )				,		num_attention_heads=params['''n_heads''']				,		num_hidden_layers=params['''n_layers''']				,		rms_norm_eps=params['''norm_eps''']				,		num_key_value_heads=UpperCamelCase				,		)
							config.save_pretrained(UpperCamelCase   )

							# Make space so we can load the model properly now.
							del state_dict
							del loaded
							gc.collect()

							print('''Loading the checkpoint in a Llama model.'''   )
							_a			  =       LlamaForCausalLM.from_pretrained(UpperCamelCase				,		torch_dtype=torch.floataa				,		low_cpu_mem_usage=UpperCamelCase   )
							# Avoid saving this as part of the config.
							del model.config._name_or_path

							print('''Saving in the Transformers format.'''   )
							model.save_pretrained(UpperCamelCase				,		safe_serialization=UpperCamelCase   )
							shutil.rmtree(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						Optional[int]				,		UpperCamelCase :						Tuple   ):





							'''simple docstring'''


							_a			  =       LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
							print(f'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'   )
							_a			  =       tokenizer_class(UpperCamelCase   )
							tokenizer.save_pretrained(UpperCamelCase   )
def 	snake_case_					():





							'''simple docstring'''


							_a			  =       argparse.ArgumentParser()
							parser.add_argument(
							    '''--input_dir'''				,		help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''				,		)
							parser.add_argument(
							    '''--model_size'''				,		choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']				,		)
							parser.add_argument(
							    '''--output_dir'''				,		help='''Location to write HF model and tokenizer'''				,		)
							parser.add_argument('''--safe_serialization'''				,		type=UpperCamelCase				,		help='''Whether or not to save using `safetensors`.'''   )
							_a			  =       parser.parse_args()
							if args.model_size != "tokenizer_only":
														write_model(
														    model_path=args.output_dir				,		input_base_path=os.path.join(args.input_dir				,		args.model_size   )				,		model_size=args.model_size				,		safe_serialization=args.safe_serialization				,		)
							_a			  =       os.path.join(args.input_dir				,		'''tokenizer.model'''   )
							write_tokenizer(args.output_dir				,		UpperCamelCase   )


if __name__ == "__main__":
				main()

