code: def _get_annotator_quality(
      labels_multiannotator: np.ndarray,
      pred_probs: np.ndarray,
      consensus_label: np.ndarray,
      num_annotations: np.ndarray,
      annotator_agreement: np.ndarray,
      model_weight: np.ndarray,
      annotator_weight: np.ndarray,
      detailed_label_quality: Optional[np.ndarray] = None,
      quality_method: str = "crowdlab",
  ) -> pd.DataFrame:
      """Returns annotator quality score for each annotator.
  
      Parameters
      ----------
      labels_multiannotator : np.ndarray
          2D numpy array of multiple given labels for each example with shape ``(N, M)``,
          where N is the number of examples and M is the number of annotators.
          For more details, labels in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`.
      pred_probs : np.ndarray
          An array of shape ``(N, K)`` of model-predicted probabilities, ``P(label=k|x)``.
          For details, predicted probabilities in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`.
      consensus_label : np.ndarray
          An array of shape ``(N,)`` with the consensus labels aggregated from all annotators.
      num_annotations : np.ndarray
          An array of shape ``(N,)`` with the number of annotators that have labeled each example.
      annotator_agreement : np.ndarray
          An array of shape ``(N,)`` with the fraction of annotators that agree with each consensus label.
      model_weight : float
          An array of shape ``(P,)`` where P is the number of models in this ensemble, specifying the model weight used in weighted averages,
          ``None`` if model weight is not used to compute quality scores
      annotator_weight : np.ndarray
          An array of shape ``(M,)`` where M is the number of annotators, specifying the annotator weights used in weighted averages,
          ``None`` if annotator weights are not used to compute quality scores
      detailed_label_quality :
          pandas DataFrame containing the detailed label quality scores for all examples and annotators
      quality_method : str, default = "crowdlab" (Options: ["crowdlab", "agreement"])
          Specifies the method used to calculate the quality of the annotators.
          For valid quality methods, view `~cleanlab.multiannotator.get_label_quality_multiannotator`
  
      Returns
      -------
      annotator_quality : np.ndarray
          Quality scores of a given annotator's labels
      """
  
      valid_methods = [
          "crowdlab",
          "agreement",
      ]
  
      if quality_method == "crowdlab":
          if detailed_label_quality is None:
              annotator_lqs = np.zeros(labels_multiannotator.shape[1])
              for i in range(len(annotator_lqs)):
                  labels = labels_multiannotator[:, i]
                  labels_mask = ~np.isnan(labels)
                  annotator_lqs[i] = np.mean(
                      get_label_quality_scores(
                          labels[labels_mask].astype(int),
                          pred_probs[labels_mask],
                      )
                  )
          else:
              annotator_lqs = np.nanmean(detailed_label_quality, axis=0)
  
          mask = num_annotations != 1
          labels_multiannotator_subset = labels_multiannotator[mask]
          consensus_label_subset = consensus_label[mask]
  
          annotator_agreement = np.zeros(
              labels_multiannotator_subset.shape[1]
          )
          for i in range(len(annotator_agreement)):
              labels = labels_multiannotator_subset[:, i]
              labels_mask = ~np.isnan(labels)
              # case where annotator does not annotate any examples with any other annotators
              # TODO: do we want to impute the mean or just return np.nan
              if np.sum(labels_mask) == 0:
                  annotator_agreement[i] = np.NaN
              else:
                  annotator_agreement[i] = np.mean(
                      labels[labels_mask]
                      == consensus_label_subset[labels_mask],
                  )
  
          avg_num_annotations_frac = np.mean(num_annotations) / len(
              annotator_weight
          )
          annotator_weight_adjusted = (
              np.sum(annotator_weight) * avg_num_annotations_frac
          )
  
          w = model_weight / (model_weight + annotator_weight_adjusted)
          annotator_quality = (
              w * annotator_lqs + (1 - w) * annotator_agreement
          )
  
      elif quality_method == "agreement":
          mask = num_annotations != 1
          labels_multiannotator_subset = labels_multiannotator[mask]
          consensus_label_subset = consensus_label[mask]
  
          annotator_quality = np.zeros(
              labels_multiannotator_subset.shape[1]
          )
          for i in range(len(annotator_quality)):
              labels = labels_multiannotator_subset[:, i]
              labels_mask = ~np.isnan(labels)
              # case where annotator does not annotate any examples with any other annotators
              if np.sum(labels_mask) == 0:
                  annotator_quality[i] = np.NaN
              else:
                  annotator_quality[i] = np.mean(
                      labels[labels_mask]
                      == consensus_label_subset[labels_mask],
                  )
  
      else:
          raise ValueError(
              f"""
              {quality_method} is not a valid annotator quality method!
              Please choose a valid quality_method: {valid_methods}
              """
          )
  
      return annotator_quality
  
imports: import warnings
  import numpy
  import pandas
  from typing import List, Dict, Any, Union, Tuple, Optional
  from cleanlab.rank import get_label_quality_scores
  from cleanlab.internal.util import get_num_classes, value_counts
  from cleanlab.internal.constants import CLIPPING_LOWER_BOUND
  from cleanlab.internal.multiannotator_utils import assert_valid_inputs_multiannotator, assert_valid_pred_probs, check_consensus_label_classes, find_best_temp_scaler, temp_scale_pred_probs
label_desc: Use image node_id 16 for calling a global function with example usage: _get_annotator_quality(labels_multiannotator, pred_probs, consensus_label, num_annotations, annotator_agreement, model_weight, annotator_weight, detailed_label_quality, quality_method) and returns: annotator_quality