code: def get_label_quality_multiannotator(
      labels_multiannotator: Union[pd.DataFrame, np.ndarray],
      pred_probs: np.ndarray,
      *,
      consensus_method: Union[str, List[str]] = "best_quality",
      quality_method: str = "crowdlab",
      calibrate_probs: bool = False,
      return_detailed_quality: bool = True,
      return_annotator_stats: bool = True,
      return_weights: bool = False,
      verbose: bool = True,
      label_quality_score_kwargs: dict = {},
  ) -> Dict[str, Any]:
      """Returns label quality scores for each example and for each annotator in a dataset labeled by multiple annotators.
  
      This function is for multiclass classification datasets where examples have been labeled by
      multiple annotators (not necessarily the same number of annotators per example).
  
      It computes one consensus label for each example that best accounts for the labels chosen by each
      annotator (and their quality), as well as a consensus quality score for how confident we are that this consensus label is actually correct.
      It also computes similar quality scores for each annotator's individual labels, and the quality of each annotator.
      Scores are between 0 and 1 (estimated via methods like CROWDLAB); lower scores indicate labels/annotators less likely to be correct.
  
      To decide what data to collect additional labels for, try the `~cleanlab.multiannotator.get_active_learning_scores`
      (ActiveLab) function, which is intended for active learning with multiple annotators.
  
      Parameters
      ----------
      labels_multiannotator : pd.DataFrame or np.ndarray
          2D pandas DataFrame or array of multiple given labels for each example with shape ``(N, M)``,
          where N is the number of examples and M is the number of annotators.
          ``labels_multiannotator[n][m]`` = label for n-th example given by m-th annotator.
  
          For a dataset with K classes, each given label must be an integer in 0, 1, ..., K-1 or ``NaN`` if this annotator did not label a particular example.
          If you have string or other differently formatted labels, you can convert them to the proper format using :py:func:`format_multiannotator_labels <cleanlab.internal.multiannotator_utils.format_multiannotator_labels>`.
          If pd.DataFrame, column names should correspond to each annotator's ID.
      pred_probs : np.ndarray
          An array of shape ``(N, K)`` of predicted class probabilities from a trained classifier model.
          Predicted probabilities in the same format expected by the :py:func:`get_label_quality_scores <cleanlab.rank.get_label_quality_scores>`.
      consensus_method : str or List[str], default = "majority_vote"
          Specifies the method used to aggregate labels from multiple annotators into a single consensus label.
          Options include:
  
          * ``majority_vote``: consensus obtained using a simple majority vote among annotators, with ties broken via ``pred_probs``.
          * ``best_quality``: consensus obtained by selecting the label with highest label quality (quality determined by method specified in ``quality_method``).
  
          A List may be passed if you want to consider multiple methods for producing consensus labels.
          If a List is passed, then the 0th element of the list is the method used to produce columns `consensus_label`, `consensus_quality_score`, `annotator_agreement` in the returned DataFrame.
          The remaning (1st, 2nd, 3rd, etc.) elements of this list are output as extra columns in the returned pandas DataFrame with names formatted as:
          `consensus_label_SUFFIX`, `consensus_quality_score_SUFFIX` where `SUFFIX` = each element of this
          list, which must correspond to a valid method for computing consensus labels.
      quality_method : str, default = "crowdlab"
          Specifies the method used to calculate the quality of the consensus label.
          Options include:
  
          * ``crowdlab``: an emsemble method that weighs both the annotators' labels as well as the model's prediction.
          * ``agreement``: the fraction of annotators that agree with the consensus label.
      calibrate_probs : bool, default = False
          Boolean value that specifies whether the provided `pred_probs` should be re-calibrated to better match the annotators' empirical label distribution.
          We recommend setting this to True in active learning applications, in order to prevent overconfident models from suggesting the wrong examples to collect labels for.
      return_detailed_quality: bool, default = True
          Boolean to specify if `detailed_label_quality` is returned.
      return_annotator_stats : bool, default = True
          Boolean to specify if `annotator_stats` is returned.
      return_weights : bool, default = False
          Boolean to specify if `model_weight` and `annotator_weight` is returned.
          Model and annotator weights are applicable for ``quality_method == crowdlab``, will return ``None`` for any other quality methods.
      verbose : bool, default = True
          Important warnings and other printed statements may be suppressed if ``verbose`` is set to ``False``.
      label_quality_score_kwargs : dict, optional
          Keyword arguments to pass into :py:func:`get_label_quality_scores <cleanlab.rank.get_label_quality_scores>`.
  
      Returns
      -------
      labels_info : dict
          Dictionary containing up to 5 pandas DataFrame with keys as below:
  
          ``label_quality`` : pandas.DataFrame
              pandas DataFrame in which each row corresponds to one example, with columns:
  
              * ``num_annotations``: the number of annotators that have labeled each example.
              * ``consensus_label``: the single label that is best for each example (you can control how it is derived from all annotators' labels via the argument: ``consensus_method``).
              * ``annotator_agreement``: the fraction of annotators that agree with the consensus label (only consider the annotators that labeled that particular example).
              * ``consensus_quality_score``: label quality score for consensus label, calculated by the method specified in ``quality_method``.
  
          ``detailed_label_quality`` : pandas.DataFrame
              Only returned if `return_detailed_quality=True`.
              Returns a pandas DataFrame with columns `quality_annotator_1`, `quality_annotator_2`, ..., `quality_annotator_M` where each entry is
              the label quality score for the labels provided by each annotator (is ``NaN`` for examples which this annotator did not label).
  
          ``annotator_stats`` : pandas.DataFrame
              Only returned if `return_annotator_stats=True`.
              Returns overall statistics about each annotator, sorted by lowest annotator_quality first.
              pandas DataFrame in which each row corresponds to one annotator (the row IDs correspond to annotator IDs), with columns:
  
              * ``annotator_quality``: overall quality of a given annotator's labels, calculated by the method specified in ``quality_method``.
              * ``num_examples_labeled``: number of examples annotated by a given annotator.
              * ``agreement_with_consensus``: fraction of examples where a given annotator agrees with the consensus label.
              * ``worst_class``: the class that is most frequently mislabeled by a given annotator.
  
          ``model_weight`` : float
              Only returned if `return_weights=True`. It is only applicable for ``quality_method == crowdlab``.
              The model weight specifies the weight of classifier model in weighted averages used to estimate label quality
              This number is an estimate of how trustworthy the model is relative the annotators.
  
          ``annotator_weight`` : np.ndarray
              Only returned if `return_weights=True`. It is only applicable for ``quality_method == crowdlab``.
              An array of shape ``(M,)`` where M is the number of annotators, specifying the weight of each annotator in weighted averages used to estimate label quality.
              These weights are estimates of how trustworthy each annotator is relative to the other annotators.
  
      """
  
      if isinstance(labels_multiannotator, pd.DataFrame):
          annotator_ids = labels_multiannotator.columns
          index_col = labels_multiannotator.index
          labels_multiannotator = (
              labels_multiannotator.replace({pd.NA: np.NaN})
              .astype(float)
              .to_numpy()
          )
      elif isinstance(labels_multiannotator, np.ndarray):
          annotator_ids = None
          index_col = None
      else:
          raise ValueError(
              "labels_multiannotator must be either a NumPy array or Pandas DataFrame."
          )
  
      if return_weights == True and quality_method != "crowdlab":
          raise ValueError(
              "Model and annotator weights are only applicable to the crowdlab quality method. "
              "Either set return_weights=False or quality_method='crowdlab'."
          )
  
      assert_valid_inputs_multiannotator(
          labels_multiannotator, pred_probs, annotator_ids=annotator_ids
      )
  
      # Count number of non-NaN values for each example
      num_annotations = np.sum(~np.isnan(labels_multiannotator), axis=1)
  
      # calibrate pred_probs
      if calibrate_probs:
          optimal_temp = find_best_temp_scaler(
              labels_multiannotator, pred_probs
          )
          pred_probs = temp_scale_pred_probs(pred_probs, optimal_temp)
  
      if not isinstance(consensus_method, list):
          consensus_method = [consensus_method]
  
      if (
          "best_quality" in consensus_method
          or "majority_vote" in consensus_method
      ):
          majority_vote_label = get_majority_vote_label(
              labels_multiannotator=labels_multiannotator,
              pred_probs=pred_probs,
              verbose=False,
          )
          (
              MV_annotator_agreement,
              MV_consensus_quality_score,
              MV_post_pred_probs,
              MV_model_weight,
              MV_annotator_weight,
          ) = _get_consensus_stats(
              labels_multiannotator=labels_multiannotator,
              pred_probs=pred_probs,
              num_annotations=num_annotations,
              consensus_label=majority_vote_label,
              quality_method=quality_method,
              verbose=verbose,
              label_quality_score_kwargs=label_quality_score_kwargs,
          )
  
      label_quality = pd.DataFrame(
          {"num_annotations": num_annotations}, index=index_col
      )
      valid_methods = ["majority_vote", "best_quality"]
      main_method = True
  
      for curr_method in consensus_method:
          # geting consensus label and stats
          if curr_method == "majority_vote":
              consensus_label = majority_vote_label
              annotator_agreement = MV_annotator_agreement
              consensus_quality_score = MV_consensus_quality_score
              post_pred_probs = MV_post_pred_probs
              model_weight = MV_model_weight
              annotator_weight = MV_annotator_weight
  
          elif curr_method == "best_quality":
              consensus_label = np.full(
                  len(majority_vote_label), np.nan
              )
              for i in range(len(consensus_label)):
                  max_pred_probs_ind = np.where(
                      MV_post_pred_probs[i]
                      == np.max(MV_post_pred_probs[i])
                  )[0]
                  if len(max_pred_probs_ind) == 1:
                      consensus_label[i] = max_pred_probs_ind[0]
                  else:
                      consensus_label[i] = majority_vote_label[i]
              consensus_label = consensus_label.astype(
                  int
              )  # convert all label types to int
  
              (
                  annotator_agreement,
                  consensus_quality_score,
                  post_pred_probs,
                  model_weight,
                  annotator_weight,
              ) = _get_consensus_stats(
                  labels_multiannotator=labels_multiannotator,
                  pred_probs=pred_probs,
                  num_annotations=num_annotations,
                  consensus_label=consensus_label,
                  quality_method=quality_method,
                  verbose=verbose,
                  label_quality_score_kwargs=label_quality_score_kwargs,
              )
  
          else:
              raise ValueError(
                  f"""
                  {curr_method} is not a valid consensus method!
                  Please choose a valid consensus_method: {valid_methods}
                  """
              )
  
          if verbose:
              # check if any classes no longer appear in the set of consensus labels
              check_consensus_label_classes(
                  labels_multiannotator=labels_multiannotator,
                  consensus_label=consensus_label,
                  consensus_method=curr_method,
              )
  
          # saving stats into dataframe, computing additional stats if specified
          if main_method:
              (
                  label_quality["consensus_label"],
                  label_quality["consensus_quality_score"],
                  label_quality["annotator_agreement"],
              ) = (
                  consensus_label,
                  consensus_quality_score,
                  annotator_agreement,
              )
  
              label_quality = label_quality.reindex(
                  columns=[
                      "consensus_label",
                      "consensus_quality_score",
                      "annotator_agreement",
                      "num_annotations",
                  ]
              )
  
              # default variable for _get_annotator_stats
              detailed_label_quality = None
  
              if return_detailed_quality:
                  # Compute the label quality scores for each annotators' labels
                  detailed_label_quality = np.apply_along_axis(
                      _get_annotator_label_quality_score,
                      axis=0,
                      arr=labels_multiannotator,
                      pred_probs=post_pred_probs,
                      label_quality_score_kwargs=label_quality_score_kwargs,
                  )
                  detailed_label_quality_df = pd.DataFrame(
                      detailed_label_quality,
                      index=index_col,
                      columns=annotator_ids,
                  ).add_prefix("quality_annotator_")
  
              if return_annotator_stats:
                  annotator_stats = _get_annotator_stats(
                      labels_multiannotator=labels_multiannotator,
                      pred_probs=post_pred_probs,
                      consensus_label=consensus_label,
                      num_annotations=num_annotations,
                      annotator_agreement=annotator_agreement,
                      model_weight=model_weight,
                      annotator_weight=annotator_weight,
                      consensus_quality_score=consensus_quality_score,
                      detailed_label_quality=detailed_label_quality,
                      annotator_ids=annotator_ids,
                      quality_method=quality_method,
                  )
  
              main_method = False
  
          else:
              (
                  label_quality[f"consensus_label_{curr_method}"],
                  label_quality[
                      f"consensus_quality_score_{curr_method}"
                  ],
                  label_quality[f"annotator_agreement_{curr_method}"],
              ) = (
                  consensus_label,
                  consensus_quality_score,
                  annotator_agreement,
              )
  
      labels_info = {
          "label_quality": label_quality,
      }
  
      if return_detailed_quality:
          labels_info[
              "detailed_label_quality"
          ] = detailed_label_quality_df
      if return_annotator_stats:
          labels_info["annotator_stats"] = annotator_stats
      if return_weights:
          labels_info["model_weight"] = model_weight
          labels_info["annotator_weight"] = annotator_weight
  
      return labels_info
  
imports: import warnings
  import numpy
  import pandas
  from typing import List, Dict, Any, Union, Tuple, Optional
  from cleanlab.rank import get_label_quality_scores
  from cleanlab.internal.util import get_num_classes, value_counts
  from cleanlab.internal.constants import CLIPPING_LOWER_BOUND
  from cleanlab.internal.multiannotator_utils import assert_valid_inputs_multiannotator, assert_valid_pred_probs, check_consensus_label_classes, find_best_temp_scaler, temp_scale_pred_probs
label_desc: Use image node_id 1 for calling a global function with example usage: get_label_quality_multiannotator(labels_multiannotator, pred_probs) and returns: labels_info