




'''simple docstring'''







import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
    from transformers import LlamaTokenizerFast
except ImportError as e:
    warnings.warn(e)
    warnings.warn(
        '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
    )
    lowerCAmelCase_     						=				None
lowerCAmelCase_     						=				{
    '''7B''': 1_1_0_0_8,
    '''13B''': 1_3_8_2_4,
    '''30B''': 1_7_9_2_0,
    '''65B''': 2_2_0_1_6,
    '''70B''': 2_8_6_7_2,
}
lowerCAmelCase_     						=				{
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase=1      ,UpperCAmelCase=256						):






   '''simple docstring'''







   return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3						)						) + multiple_of - 1) // multiple_of)







def 						_A      ( UpperCAmelCase						):






   '''simple docstring'''







   with open(UpperCAmelCase      ,'r'						) as f:
      return json.load(UpperCAmelCase						)







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase						):






   '''simple docstring'''







   with open(UpperCAmelCase      ,'w'						) as f:
      json.dump(UpperCAmelCase      ,UpperCAmelCase						)







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase=True						):






   '''simple docstring'''







   os.makedirs(UpperCAmelCase      ,exist_ok=UpperCAmelCase						)
   A__       =     os.path.join(UpperCAmelCase      ,'tmp'						)
   os.makedirs(UpperCAmelCase      ,exist_ok=UpperCAmelCase						)

   A__       =     read_json(os.path.join(UpperCAmelCase      ,'params.json'						)						)
   A__       =     NUM_SHARDS[model_size]
   A__       =     params['n_layers']
   A__       =     params['n_heads']
   A__       =     n_heads // num_shards
   A__       =     params['dim']
   A__       =     dim // n_heads
   A__       =     1_00_00.0
   A__       =     1.0 / (base ** (torch.arange(0      ,UpperCAmelCase      ,2						).float() / dims_per_head))

   if "n_kv_heads" in params:
      A__       =     params['n_kv_heads']  # for GQA / MQA
      A__       =     n_heads_per_shard // num_key_value_heads
      A__       =     dim // num_key_value_heads
   else:  # compatibility with other checkpoints
      A__       =     n_heads
      A__       =     n_heads_per_shard
      A__       =     dim

   # permute for sliced rotary
   def permute(UpperCAmelCase      ,UpperCAmelCase=n_heads      ,UpperCAmelCase=dim      ,UpperCAmelCase=dim						):
      return w.view(UpperCAmelCase      ,dima // n_heads // 2      ,2      ,UpperCAmelCase						).transpose(1      ,2						).reshape(UpperCAmelCase      ,UpperCAmelCase						)

   print(F"""Fetching all parameters from the checkpoint at {input_base_path}."""						)
   # Load weights
   if model_size == "7B":
      # Not sharded
      # (The sharded implementation would also work, but this is simpler.)
      A__       =     torch.load(os.path.join(UpperCAmelCase      ,'consolidated.00.pth'						)      ,map_location='cpu'						)
   else:
      # Sharded
      A__       =     [
          torch.load(os.path.join(UpperCAmelCase      ,F"""consolidated.{i:02d}.pth"""						)      ,map_location='cpu'						)
          for i in range(UpperCAmelCase						)
      ]
   A__       =     0
   A__       =     {'weight_map': {}}
   for layer_i in range(UpperCAmelCase						):
      A__       =     F"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
      if model_size == "7B":
         # Unsharded
         A__       =     {
             F"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
                 loaded[F"""layers.{layer_i}.attention.wq.weight"""]						),
             F"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
                 loaded[F"""layers.{layer_i}.attention.wk.weight"""]						),
             F"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[F"""layers.{layer_i}.attention.wv.weight"""],
             F"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[F"""layers.{layer_i}.attention.wo.weight"""],
             F"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w1.weight"""],
             F"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w2.weight"""],
             F"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w3.weight"""],
             F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[F"""layers.{layer_i}.attention_norm.weight"""],
             F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[F"""layers.{layer_i}.ffn_norm.weight"""],
         }
      else:
         # Sharded
         # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
         # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
         # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

         A__       =     {
             F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
                 F"""layers.{layer_i}.attention_norm.weight"""
             ].clone(),
             F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
                 F"""layers.{layer_i}.ffn_norm.weight"""
             ].clone(),
         }
         A__       =     permute(
             torch.cat(
                 [
                     loaded[i][F"""layers.{layer_i}.attention.wq.weight"""].view(UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase						)
                     for i in range(UpperCAmelCase						)
                 ]      ,dim=0      ,).reshape(UpperCAmelCase      ,UpperCAmelCase						)						)
         A__       =     permute(
             torch.cat(
                 [
                     loaded[i][F"""layers.{layer_i}.attention.wk.weight"""].view(
                         UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase						)
                     for i in range(UpperCAmelCase						)
                 ]      ,dim=0      ,).reshape(UpperCAmelCase      ,UpperCAmelCase						)      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,)
         A__       =     torch.cat(
             [
                 loaded[i][F"""layers.{layer_i}.attention.wv.weight"""].view(
                     UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase						)
                 for i in range(UpperCAmelCase						)
             ]      ,dim=0      ,).reshape(UpperCAmelCase      ,UpperCAmelCase						)

         A__       =     torch.cat(
             [loaded[i][F"""layers.{layer_i}.attention.wo.weight"""] for i in range(UpperCAmelCase						)]      ,dim=1						)
         A__       =     torch.cat(
             [loaded[i][F"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(UpperCAmelCase						)]      ,dim=0						)
         A__       =     torch.cat(
             [loaded[i][F"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(UpperCAmelCase						)]      ,dim=1						)
         A__       =     torch.cat(
             [loaded[i][F"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(UpperCAmelCase						)]      ,dim=0						)

      A__       =     inv_freq
      for k, v in state_dict.items():
         A__       =     filename
         param_count += v.numel()
      torch.save(UpperCAmelCase      ,os.path.join(UpperCAmelCase      ,UpperCAmelCase						)						)

   A__       =     F"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
   if model_size == "7B":
      # Unsharded
      A__       =     {
          'model.embed_tokens.weight': loaded['tok_embeddings.weight'],
          'model.norm.weight': loaded['norm.weight'],
          'lm_head.weight': loaded['output.weight'],
      }
   else:
      A__       =     {
          'model.norm.weight': loaded[0]['norm.weight'],
          'model.embed_tokens.weight': torch.cat(
              [loaded[i]['tok_embeddings.weight'] for i in range(UpperCAmelCase						)]      ,dim=1						),
          'lm_head.weight': torch.cat([loaded[i]['output.weight'] for i in range(UpperCAmelCase						)]      ,dim=0						),
      }

   for k, v in state_dict.items():
      A__       =     filename
      param_count += v.numel()
   torch.save(UpperCAmelCase      ,os.path.join(UpperCAmelCase      ,UpperCAmelCase						)						)

   # Write configs
   A__       =     {'total_size': param_count * 2}
   write_json(UpperCAmelCase      ,os.path.join(UpperCAmelCase      ,'pytorch_model.bin.index.json'						)						)
   A__       =     params['ffn_dim_multiplier'] if 'ffn_dim_multiplier' in params else 1
   A__       =     params['multiple_of'] if 'multiple_of' in params else 256
   A__       =     LlamaConfig(
       hidden_size=UpperCAmelCase      ,intermediate_size=compute_intermediate_size(UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase						)      ,num_attention_heads=params['n_heads']      ,num_hidden_layers=params['n_layers']      ,rms_norm_eps=params['norm_eps']      ,num_key_value_heads=UpperCAmelCase      ,)
   config.save_pretrained(UpperCAmelCase						)

   # Make space so we can load the model properly now.
   del state_dict
   del loaded
   gc.collect()

   print('Loading the checkpoint in a Llama model.'						)
   A__       =     LlamaForCausalLM.from_pretrained(UpperCAmelCase      ,torch_dtype=torch.floataa      ,low_cpu_mem_usage=UpperCAmelCase						)
   # Avoid saving this as part of the config.
   del model.config._name_or_path

   print('Saving in the Transformers format.'						)
   model.save_pretrained(UpperCAmelCase      ,safe_serialization=UpperCAmelCase						)
   shutil.rmtree(UpperCAmelCase						)







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase						):






   '''simple docstring'''







   A__       =     LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
   print(F"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""						)
   A__       =     tokenizer_class(UpperCAmelCase						)
   tokenizer.save_pretrained(UpperCAmelCase						)







def 						_A      ( ):






   '''simple docstring'''







   A__       =     argparse.ArgumentParser()
   parser.add_argument(
       '--input_dir'      ,help='Location of LLaMA weights, which contains tokenizer.model and model folders'      ,)
   parser.add_argument(
       '--model_size'      ,choices=['7B', '7Bf', '13B', '13Bf', '30B', '65B', '70B', '70Bf', 'tokenizer_only']      ,)
   parser.add_argument(
       '--output_dir'      ,help='Location to write HF model and tokenizer'      ,)
   parser.add_argument('--safe_serialization'      ,type=UpperCAmelCase      ,help='Whether or not to save using `safetensors`.'						)
   A__       =     parser.parse_args()
   if args.model_size != "tokenizer_only":
      write_model(
          model_path=args.output_dir      ,input_base_path=os.path.join(args.input_dir      ,args.model_size						)      ,model_size=args.model_size      ,safe_serialization=args.safe_serialization      ,)
   A__       =     os.path.join(args.input_dir      ,'tokenizer.model'						)
   write_tokenizer(args.output_dir      ,UpperCAmelCase						)


if __name__ == "__main__":
    main()








'''simple docstring'''







import itertools
import json
import linecache
import os
import pickle
import re
import socket
import string
from collections import Counter
from logging import getLogger
from pathlib import Path
from typing import Callable, Dict, Iterable, List

import git
import torch
from torch.utils.data import Dataset

from transformers import BartTokenizer, RagTokenizer, TaTokenizer







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase=True      ,UpperCAmelCase="pt"						):






   '''simple docstring'''







   A__       =     {'add_prefix_space': True} if isinstance(UpperCAmelCase      ,UpperCAmelCase						) and not line.startswith(' '						) else {}
   A__       =     padding_side
   return tokenizer(
       [line]      ,max_length=UpperCAmelCase      ,padding='max_length' if pad_to_max_length else None      ,truncation=UpperCAmelCase      ,return_tensors=UpperCAmelCase      ,add_special_tokens=UpperCAmelCase      ,**UpperCAmelCase      ,)







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase=None      ,):






   '''simple docstring'''







   A__       =     input_ids.ne(UpperCAmelCase						).any(dim=0						)
   if attention_mask is None:
      return input_ids[:, keep_column_mask]
   else:
      return (input_ids[:, keep_column_mask], attention_mask[:, keep_column_mask])

class 	_snake_case(			UpperCAmelCase      ):






 def __init__(self       :  List[str]    ,		a       :  Any    ,		a       :  Union[str, Any]    ,		a       :  int    ,		a       :  Optional[int]    ,		a       :  Optional[Any]="train"    ,		a       :  Optional[int]=None    ,		a       :  Tuple=None    ,		a       :  str=None    ,		a       :  Dict=""    ,		)       ->   Dict:






    """simple docstring"""


    super().__init__()
    A__       =     Path(a    ).joinpath(type_path + '.source'    )
    A__       =     Path(a    ).joinpath(type_path + '.target'    )
    A__       =     self.get_char_lens(self.src_file    )
    A__       =     max_source_length
    A__       =     max_target_length
    assert min(self.src_lens    ) > 0, f"""found empty line in {self.src_file}"""
    A__       =     tokenizer
    A__       =     prefix
    if n_obs is not None:
       A__       =     self.src_lens[:n_obs]
    A__       =     src_lang
    A__       =     tgt_lang






 def __len__(self       :  Optional[Any]    )       ->   List[str]:






    """simple docstring"""


    return len(self.src_lens    )






 def __getitem__(self       :  Any    ,		a       :  str    )       ->   Dict[str, torch.Tensor]:






    """simple docstring"""


    A__       =     index + 1  # linecache starts at 1
    A__       =     self.prefix + linecache.getline(str(self.src_file    )    ,		a    ).rstrip('\n'    )
    A__       =     linecache.getline(str(self.tgt_file    )    ,		a    ).rstrip('\n'    )
    assert source_line, f"""empty source line for index {index}"""
    assert tgt_line, f"""empty tgt line for index {index}"""

    # Need to add eos token manually for T5
    if isinstance(self.tokenizer    ,		a    ):
       source_line += self.tokenizer.eos_token
       tgt_line += self.tokenizer.eos_token

    # Pad source and target to the right
    A__       =     (
        self.tokenizer.question_encoder if isinstance(self.tokenizer    ,		a    ) else self.tokenizer
    )
    A__       =     self.tokenizer.generator if isinstance(self.tokenizer    ,		a    ) else self.tokenizer

    A__       =     encode_line(a    ,		a    ,		self.max_source_length    ,		'right'    )
    A__       =     encode_line(a    ,		a    ,		self.max_target_length    ,		'right'    )

    A__       =     source_inputs['input_ids'].squeeze()
    A__       =     target_inputs['input_ids'].squeeze()
    A__       =     source_inputs['attention_mask'].squeeze()
    return {
        "input_ids": source_ids,
        "attention_mask": src_mask,
        "decoder_input_ids": target_ids,
    }






 @staticmethod
 def 						_UpperCamelCase (a       :  Tuple    )       ->   Tuple:






    """simple docstring"""


    return [len(a    ) for x in Path(a    ).open().readlines()]






 def 						_UpperCamelCase (self       :  str    ,		a       :  int    )       ->   Dict[str, torch.Tensor]:






    """simple docstring"""


    A__       =     torch.stack([x['input_ids'] for x in batch]    )
    A__       =     torch.stack([x['attention_mask'] for x in batch]    )
    A__       =     torch.stack([x['decoder_input_ids'] for x in batch]    )
    A__       =     (
        self.tokenizer.generator.pad_token_id
        if isinstance(self.tokenizer    ,		a    )
        else self.tokenizer.pad_token_id
    )
    A__       =     (
        self.tokenizer.question_encoder.pad_token_id
        if isinstance(self.tokenizer    ,		a    )
        else self.tokenizer.pad_token_id
    )
    A__       =     trim_batch(a    ,		a    )
    A__  ,			A__       =     trim_batch(a    ,		a    ,		attention_mask=a    )
    A__       =     {
        'input_ids': source_ids,
        'attention_mask': source_mask,
        'decoder_input_ids': y,
    }
    return batch




lowerCAmelCase_     						=				getLogger(__name__)







def 						_A      ( UpperCAmelCase						):






   '''simple docstring'''







   return list(itertools.chain.from_iterable(UpperCAmelCase						)						)







def 						_A      ( UpperCAmelCase						):






   '''simple docstring'''







   A__       =     get_git_info()
   save_json(UpperCAmelCase      ,os.path.join(UpperCAmelCase      ,'git_log.json'						)						)







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase=4      ,**UpperCAmelCase						):






   '''simple docstring'''







   with open(UpperCAmelCase      ,'w'						) as f:
      json.dump(UpperCAmelCase      ,UpperCAmelCase      ,indent=UpperCAmelCase      ,**UpperCAmelCase						)







def 						_A      ( UpperCAmelCase						):






   '''simple docstring'''







   with open(UpperCAmelCase						) as f:
      return json.load(UpperCAmelCase						)







def 						_A      ( ):






   '''simple docstring'''







   A__       =     git.Repo(search_parent_directories=UpperCAmelCase						)
   A__       =     {
       'repo_id': str(UpperCAmelCase						),
       'repo_sha': str(repo.head.object.hexsha						),
       'repo_branch': str(repo.active_branch						),
       'hostname': str(socket.gethostname()						),
   }
   return repo_infos







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase						):






   '''simple docstring'''







   return list(map(UpperCAmelCase      ,UpperCAmelCase						)						)







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase						):






   '''simple docstring'''







   with open(UpperCAmelCase      ,'wb'						) as f:
      return pickle.dump(UpperCAmelCase      ,UpperCAmelCase						)







def 						_A      ( UpperCAmelCase						):






   '''simple docstring'''







   def remove_articles(UpperCAmelCase						):
      return re.sub(r'\b(a|an|the)\b'      ,' '      ,UpperCAmelCase						)

   def white_space_fix(UpperCAmelCase						):
      return " ".join(text.split()						)

   def remove_punc(UpperCAmelCase						):
      A__       =     set(string.punctuation						)
      return "".join(ch for ch in text if ch not in exclude						)

   def lower(UpperCAmelCase						):
      return text.lower()

   return white_space_fix(remove_articles(remove_punc(lower(UpperCAmelCase						)						)						)						)







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase						):






   '''simple docstring'''







   A__       =     normalize_answer(UpperCAmelCase						).split()
   A__       =     normalize_answer(UpperCAmelCase						).split()
   A__       =     Counter(UpperCAmelCase						) & Counter(UpperCAmelCase						)
   A__       =     sum(common.values()						)
   if num_same == 0:
      return 0
   A__       =     1.0 * num_same / len(UpperCAmelCase						)
   A__       =     1.0 * num_same / len(UpperCAmelCase						)
   A__       =     (2 * precision * recall) / (precision + recall)
   return fa







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase						):






   '''simple docstring'''







   return normalize_answer(UpperCAmelCase						) == normalize_answer(UpperCAmelCase						)







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase						):






   '''simple docstring'''







   assert len(UpperCAmelCase						) == len(UpperCAmelCase						)
   A__       =     0
   for hypo, pred in zip(UpperCAmelCase      ,UpperCAmelCase						):
      em += exact_match_score(UpperCAmelCase      ,UpperCAmelCase						)
   if len(UpperCAmelCase						) > 0:
      em /= len(UpperCAmelCase						)
   return {"em": em}







def 						_A      ( UpperCAmelCase						):






   '''simple docstring'''







   return model_prefix.startswith('rag'						)







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase						):






   '''simple docstring'''







   A__       =     {p: p for p in extra_params}
   # T5 models don't have `dropout` param, they have `dropout_rate` instead
   A__       =     'dropout_rate'
   for p in extra_params:
      if getattr(UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase						):
         if not hasattr(UpperCAmelCase      ,UpperCAmelCase						) and not hasattr(UpperCAmelCase      ,equivalent_param[p]						):
            logger.info('config doesn\'t have a `{}` attribute'.format(UpperCAmelCase						)						)
            delattr(UpperCAmelCase      ,UpperCAmelCase						)
            continue
         A__       =     p if hasattr(UpperCAmelCase      ,UpperCAmelCase						) else equivalent_param[p]
         setattr(UpperCAmelCase      ,UpperCAmelCase      ,getattr(UpperCAmelCase      ,UpperCAmelCase						)						)
         delattr(UpperCAmelCase      ,UpperCAmelCase						)
   return hparams, config


