docstr: Hierarchical LIME
  
  model_wrapped: should contain get_conditional_loglikelihood_batch(texts, completion_offset)
  max_parts_to_perturb: maximum parts to perturb for a single instance
  n_samples: number of total perturbations
  prompt_fn: whether to format the prompt after perturbations
  
  This will output the final attribution scores for the parts along with the entire importance cache
  of the hierarchy
docstr_tok: ["Hierarchical","LIME","model_wrapped",":","should","contain","get_conditional_loglikelihood_batch","(","texts",",","completion_offset",")","max_parts_to_perturb",":","maximum","parts","to","perturb","for","a","single","instance","n_samples",":","number","of","total","perturbations","prompt_fn",":","whether","to","format","the","prompt","after","perturbations","This","will","output","the","final","attribution","scores","for","the","parts","along","with","the","entire","importance","cache","of","the","hierarchy"]
code: def attribution(
      self,
      model_wrapped,
      prompt,
      completion,
      max_parts_to_perturb=3,
      n_samples=100,
      max_features=3,
      prompt_fn=None,
      intermediate_log_fn=None,
      *args,
      **kwargs,
  ):
      """
      Hierarchical LIME
  
      model_wrapped: should contain get_conditional_loglikelihood_batch(texts, completion_offset)
      max_parts_to_perturb: maximum parts to perturb for a single instance
      n_samples: number of total perturbations
      prompt_fn: whether to format the prompt after perturbations
  
      This will output the final attribution scores for the parts along with the entire importance cache
      of the hierarchy
      """
      # TODO: Add a smarter strategy to pick number of features.
      # e.g. Can say each feature should be perturbed at least k times in
      # expectation.
  
      parts = None
      parts_to_replace = None
      importance_cache = {}
      for step_idx, perturbation_specs in enumerate(
          self.perturbation_hierarchy
      ):
          max_feats_step = perturbation_specs["max_features"]
          perturbation_specs.pop("max_features")
          print(f"Hierarchy Step: {step_idx}")
          local_attribution = self.step_explainer(
              perturbation_model=self.perturbation_model,
              progress_bar=self.progress_bar,
              **perturbation_specs,
          )
          if parts is not None:
              # Construct new parts
              temp_parts = []
              temp_parts_to_replace = []
              parts_idx = 0
              # Split each larger part into subparts with the next partition
              # in the hierarchy
              for i, part in enumerate(parts):
                  _temp_parts = local_attribution.partition(part)
                  temp_parts.extend(_temp_parts)
                  # Mark the new parts to replace
                  if i in parts_to_replace:
                      temp_parts_to_replace.extend(
                          [
                              parts_idx + j
                              for j in range(len(_temp_parts))
                          ]
                      )
                  parts_idx += len(_temp_parts)
  
              parts = temp_parts
              parts_to_replace = temp_parts_to_replace
  
          # Perform the attribution
          attribution_scores, parts = local_attribution.attribution(
              model_wrapped,
              prompt,
              completion,
              max_parts_to_perturb=max_parts_to_perturb,
              n_samples=n_samples,
              max_features=max_feats_step,
              parts=parts,
              parts_to_replace=parts_to_replace,
              prompt_fn=prompt_fn,
          )
  
          # Take important features and only replace them in the future
          parts_to_replace = sorted([s[0] for s in attribution_scores])
          importance_cache[step_idx] = [
              parts,
              parts_to_replace,
              attribution_scores,
          ]
          if intermediate_log_fn is not None:
              step_idx = f"{step_idx+1}: {perturbation_specs}"
              intermediate_log_fn(
                  [parts, parts_to_replace, attribution_scores],
                  step_idx,
              )
      return importance_cache
  
imports: import torch
  import numpy
  from abc import ABC, abstractmethod
  from sklearn.linear_model import Ridge, lars_path
  from .text_utils import *
  from .perturbation_tools import RemovalPerturbation
label_desc: Use image node_id 2 for calling the LocalExplanationHierarchical obj's underlying member method code with example usage: obj.attribution(model_wrapped, prompt, completion, max_parts_to_perturb, n_samples, max_features, prompt_fn, intermediate_log_fn) and returns: importance_cache