code: def test_label_quality_scores_multiannotator():
      labels = data["labels"]
      pred_probs = data["pred_probs"]
  
      multiannotator_dict = get_label_quality_multiannotator(
          labels, pred_probs
      )
      assert isinstance(multiannotator_dict, dict)
      assert len(multiannotator_dict) == 3
  
      label_quality_multiannotator = multiannotator_dict[
          "label_quality"
      ]
      assert isinstance(label_quality_multiannotator, pd.DataFrame)
      assert len(label_quality_multiannotator) == len(labels)
      assert all(label_quality_multiannotator["num_annotations"] > 0)
      assert set(
          label_quality_multiannotator["consensus_label"]
      ).issubset(np.unique(labels))
      assert all(
          (label_quality_multiannotator["annotator_agreement"] >= 0)
          & (label_quality_multiannotator["annotator_agreement"] <= 1)
      )
      assert all(
          (label_quality_multiannotator["consensus_quality_score"] >= 0)
          & (
              label_quality_multiannotator["consensus_quality_score"]
              <= 1
          )
      )
  
      annotator_stats = multiannotator_dict["annotator_stats"]
      assert isinstance(annotator_stats, pd.DataFrame)
      assert len(annotator_stats) == labels.shape[1]
      assert all(
          (annotator_stats["annotator_quality"] >= 0)
          & (annotator_stats["annotator_quality"] <= 1)
      )
      assert all(annotator_stats["num_examples_labeled"] > 0)
      assert all(
          (annotator_stats["agreement_with_consensus"] >= 0)
          & (annotator_stats["agreement_with_consensus"] <= 1)
      )
      assert set(annotator_stats["worst_class"]).issubset(
          np.unique(labels)
      )
  
      detailed_label_quality = multiannotator_dict[
          "detailed_label_quality"
      ]
      assert detailed_label_quality.shape == labels.shape
  
      # test verbose=False
      multiannotator_dict = get_label_quality_multiannotator(
          labels, pred_probs, verbose=False
      )
  
      # test passing a list into consensus_method
      multiannotator_dict = get_label_quality_multiannotator(
          labels,
          pred_probs,
          consensus_method=["majority_vote", "best_quality"],
      )
  
      # test passing arguments for get_label_quality_scores
      multiannotator_dict = get_label_quality_multiannotator(
          labels,
          pred_probs,
          label_quality_score_kwargs={"method": "normalized_margin"},
      )
  
      # test different quality_methods
      # also testing passing labels as np.ndarray
      multiannotator_dict = get_label_quality_multiannotator(
          np.array(labels), pred_probs, quality_method="agreement"
      )
  
      # test returning annotator_stats
      multiannotator_dict = get_label_quality_multiannotator(
          labels, pred_probs, return_annotator_stats=False
      )
      assert isinstance(multiannotator_dict, dict)
      assert len(multiannotator_dict) == 2
      assert isinstance(
          multiannotator_dict["label_quality"], pd.DataFrame
      )
      assert isinstance(
          multiannotator_dict["detailed_label_quality"], pd.DataFrame
      )
  
      # test returning detailed_label_quality
      multiannotator_dict = get_label_quality_multiannotator(
          labels, pred_probs, return_detailed_quality=False
      )
      assert isinstance(multiannotator_dict, dict)
      assert len(multiannotator_dict) == 2
      assert isinstance(
          multiannotator_dict["label_quality"], pd.DataFrame
      )
      assert isinstance(
          multiannotator_dict["annotator_stats"], pd.DataFrame
      )
  
      # test return detailed and annotator stats
      multiannotator_dict = get_label_quality_multiannotator(
          labels,
          pred_probs,
          return_detailed_quality=False,
          return_annotator_stats=False,
      )
      assert isinstance(multiannotator_dict, dict)
      assert len(multiannotator_dict) == 1
      assert isinstance(
          multiannotator_dict["label_quality"], pd.DataFrame
      )
  
      # test return model and annotator weights
      multiannotator_dict = get_label_quality_multiannotator(
          labels, pred_probs, return_weights=True
      )
      assert len(multiannotator_dict) == 5
      assert isinstance(multiannotator_dict["model_weight"], float)
      assert isinstance(
          multiannotator_dict["annotator_weight"], np.ndarray
      )
  
      # test non-numeric annotator names
      labels_string_names = labels.add_prefix("anno_")
      multiannotator_dict = get_label_quality_multiannotator(
          labels_string_names, pred_probs, return_detailed_quality=False
      )
  
      # test calibration
      multiannotator_dict = get_label_quality_multiannotator(
          labels, pred_probs, calibrate_probs=True
      )
  
      # test incorrect consensus_method
      try:
          multiannotator_dict = get_label_quality_multiannotator(
              labels, pred_probs, consensus_method="fake_method"
          )
      except ValueError as e:
          assert "not a valid consensus method" in str(e)
  
      # test error when return_weights == True and quality_method != "crowdlab"
      try:
          multiannotator_dict = get_label_quality_multiannotator(
              labels,
              pred_probs,
              return_weights=True,
              quality_method="agreement",
          )
      except ValueError as e:
          assert (
              "Model and annotator weights are only applicable to the crowdlab quality method"
              in str(e)
          )
  
      # test error catching when labels_multiannotator has NaN columns
      labels_NA = deepcopy(labels_string_names)
      labels_NA["anno_0"] = pd.NA
      try:
          multiannotator_dict = get_label_quality_multiannotator(
              labels_NA,
              pred_probs,
          )
      except ValueError as e:
          assert "cannot have columns with all NaN" in str(e)
          assert (
              "Annotators ['anno_0'] did not label any examples."
              in str(e)
          )
  
      # try same thing as above but with numpy array
      labels_nan = deepcopy(labels).values.astype(float)
      labels_nan[:, 1] = np.NaN
      try:
          multiannotator_dict = get_label_quality_multiannotator(
              labels_nan,
              pred_probs,
          )
      except ValueError as e:
          assert "cannot have columns with all NaN" in str(e)
          assert "Annotators [1] did not label any examples." in str(e)
  
      # test error catching when labels_multiannotator has NaN rows
      labels_nan = pd.DataFrame(
          [
              [0, np.NaN, np.NaN],
              [np.NaN, 1, np.NaN],
              [np.NaN, np.NaN, 2],
              [np.NaN, np.NaN, np.NaN],
              [np.NaN, np.NaN, 2],
          ]
      )
      pred_probs = np.random.random((5, 3))
  
      try:
          multiannotator_dict = get_label_quality_multiannotator(
              labels_nan,
              pred_probs,
          )
      except ValueError as e:
          assert "cannot have rows with all NaN" in str(e)
          assert "Examples [3] do not have any labels." in str(e)
  
      # test error when using wrong function
      try:
          multiannotator_dict = get_label_quality_multiannotator(
              labels,
              np.array([pred_probs, pred_probs]),
              return_weights=True,
          )
      except ValueError as e:
          assert "use the ensemble version of this function" in str(e)
  
      # make sure error is thrown if labels are not 2D
      labels_flat = labels.values[:, 0].flatten()
      print(labels_flat.ndim)
      print(labels_flat)
      try:
          multiannotator_dict = get_label_quality_multiannotator(
              labels_flat, pred_probs
          )
      except ValueError as e:
          assert (
              "labels_multiannotator must be a 2D array or dataframe"
              in str(e)
          )
  
imports: import numpy
  import pytest
  from copy import deepcopy
  from cleanlab.benchmarking.noise_generation import generate_noise_matrix_from_trace
  from cleanlab.benchmarking.noise_generation import generate_noisy_labels
  from cleanlab import count
  from cleanlab.multiannotator import get_label_quality_multiannotator, get_label_quality_multiannotator_ensemble, get_active_learning_scores, get_active_learning_scores_ensemble, get_majority_vote_label, convert_long_to_wide_dataset
  from cleanlab.internal.multiannotator_utils import format_multiannotator_labels
  import pandas
  from sklearn.linear_model import LogisticRegression
label_desc: Use image node_id 5 for calling a global function with example usage: test_label_quality_scores_multiannotator() without return types