code: def _remove_duplicate_names(
      state_dict: Dict[str, torch.Tensor],
      *,
      preferred_names: List[str] = None,
      discard_names: List[str] = None,
  ) -> Dict[str, List[str]]:
      if preferred_names is None:
          preferred_names = []
      preferred_names = set(preferred_names)
      if discard_names is None:
          discard_names = []
      discard_names = set(discard_names)
  
      shareds = _find_shared_tensors(state_dict)
      to_remove = defaultdict(list)
      for shared in shareds:
          complete_names = set(
              [
                  name
                  for name in shared
                  if _is_complete(state_dict[name])
              ]
          )
          if not complete_names:
              if len(shared) == 1:
                  # Force contiguous
                  name = list(shared)[0]
                  state_dict[name] = state_dict[name].clone()
                  complete_names = {name}
              else:
                  raise RuntimeError(
                      f"Error while trying to find names to remove to save state dict, but found no suitable name to keep for saving amongst: {shared}. None is covering the entire storage.Refusing to save/load the model since you could be storing much more memory than needed. Please refer to https://huggingface.co/docs/safetensors/torch_shared_tensors for more information. Or open an issue."
                  )
  
          keep_name = sorted(list(complete_names))[0]
  
          # Mecanism to preferentially select keys to keep
          # coming from the on-disk file to allow
          # loading models saved with a different choice
          # of keep_name
          preferred = complete_names.difference(discard_names)
          if preferred:
              keep_name = sorted(list(preferred))[0]
  
          if preferred_names:
              preferred = preferred_names.intersection(complete_names)
              if preferred:
                  keep_name = sorted(list(preferred))[0]
          for name in sorted(shared):
              if name != keep_name:
                  to_remove[keep_name].append(name)
      return to_remove
  
file_path: text-generation-inference/server/text_generation_server/utils/convert.py
imports: import datetime
  import torch
  import os
  from loguru import logger
  from pathlib import Path
  from safetensors.torch import save_file, load_file, _find_shared_tensors, _is_complete
  from typing import List, Dict
  from collections import defaultdict
label_desc: Use image node_id 1 for calling a global function with example usage: _remove_duplicate_names(state_dict) and returns: to_remove