class_docstr: Determines optimal thresholds for classifiers given constraints.
class_docstr_tok: ["Determines","optimal","thresholds","for","classifiers","given","constraints","."]
docstr: Calculates optimal thresholds on the provided data.
  
  Args:
    inputs: all examples in the dataset, in the indexed input format.
    model: the model being explained.
    dataset: the dataset which the current examples belong to.
    model_outputs: optional model outputs from calling model.predict(inputs).
    config: a config which should specify TresholderConfig options.
  
  Returns:
    A JsonDict containing the calcuated thresholds
docstr_tok: ["Calculates","optimal","thresholds","on","the","provided","data",".","Args",":","inputs",":","all","examples","in","the","dataset",",","in","the","indexed","input","format",".","model",":","the","model","being","explained",".","dataset",":","the","dataset","which","the","current","examples","belong","to",".","model_outputs",":","optional","model","outputs","from","calling","model.predict","(","inputs",")",".","config",":","a","config","which","should","specify","TresholderConfig","options",".","Returns",":","A","JsonDict","containing","the","calcuated","thresholds"]
code: def run(
      self,
      inputs: Sequence[JsonDict],
      model: lit_model.Model,
      dataset: lit_dataset.Dataset,
      model_outputs: Optional[list[JsonDict]] = None,
      config: Optional[JsonDict] = None,
  ) -> Optional[list[JsonDict]]:
      """Calculates optimal thresholds on the provided data.
  
      Args:
        inputs: all examples in the dataset, in the indexed input format.
        model: the model being explained.
        dataset: the dataset which the current examples belong to.
        model_outputs: optional model outputs from calling model.predict(inputs).
        config: a config which should specify TresholderConfig options.
  
      Returns:
        A JsonDict containing the calcuated thresholds
      """
      # Convert any facets from using IndexedInputs to using JsonDicts prior to
      # passing to self.run().
      if config and (facets := config.get("facets")):
          for facet_dict in facets.values():
              if facet_data := facet_dict.get("data"):
                  facet_dict["data"] = [
                      ex.get("data", ex) for ex in facet_data
                  ]
  
      config = ThresholderConfig(**(config or {}))
  
      pred_keys = []
      for pred_key, pred_spec in model.output_spec().items():
          if not isinstance(pred_spec, types.MulticlassPreds):
              continue
  
          if not (parent_key := pred_spec.parent):
              continue
  
          parent_spec: Optional[types.LitType] = dataset.spec().get(
              parent_key
          )
          if self.metrics_gen.is_field_compatible(
              pred_spec, parent_spec
          ):
              pred_keys.append(pred_key)
  
      indexed_outputs = {
          ex["_id"]: output
          for (ex, output) in zip(inputs, model_outputs)
      }
  
      # Try all margins for thresholds from 0 to 1, by hundreths.
      margins_to_try = [
          self.threshold_to_margin(t) for t in np.linspace(0, 1, 101)
      ]
  
      # Get binary classification metrics for all margins, for the entire
      # dataset, and also for each facet specified in the config.
      dataset_results = []
      faceted_results = {}
      # Loop over each margin/threshold to check.
      for margin in margins_to_try:
          # Set up an empty config to pass to the metrics generator.
          metrics_config = {}
          for pred_key in pred_keys:
              metrics_config[pred_key] = {"": {"margin": margin}}
  
          # Get and store the metrics for the entire dataset for this margin.
          dataset_results.append(
              self.metrics_gen.run(
                  list(inputs),
                  model,
                  dataset,
                  model_outputs,
                  metrics_config,
              )
          )
  
          # Get and store the metrics for each facet of the dataset for this margin.
          if not (facets := config.facets):
              continue
  
          for facet_key, facet_dict in facets.items():
              if not (facet_data := facet_dict.get("data")):
                  continue
  
              if facet_key not in faceted_results:
                  faceted_results[facet_key] = []
              faceted_outputs = [
                  indexed_outputs[ex["_id"]] for ex in facet_data
              ]
              faceted_results[facet_key].append(
                  self.metrics_gen.run(
                      facet_data,
                      model,
                      dataset,
                      faceted_outputs,
                      metrics_config,
                  )
              )
  
      pred_keys = [result["pred_key"] for result in dataset_results[0]]
      ret = []
  
      # Find threshold information for each prediction key.
      for i, pred_key in enumerate(pred_keys):
          ret.append(
              self.get_thresholds_for_pred_key(
                  pred_key,
                  i,
                  margins_to_try,
                  dataset_results,
                  faceted_results,
                  config,
              )
          )
      return ret
  
imports: from collections.abc import Mapping, Sequence
  import math
  from typing import Optional
  import attr
  from lit_nlp.api import components
  from lit_nlp.api import dataset
  from lit_nlp.api import model
  from lit_nlp.api import types
  from lit_nlp.components import metrics
  from lit_nlp.lib import utils
  import numpy
label_desc: Use image node_id 6 for calling the Thresholder obj's underlying member method code with example usage: obj.run(inputs, model, dataset, model_outputs, config) and returns: ret