code: def _flip_rate(
      counterfactual_prediction_key: Optional[str] = None,
      example_id_key: Optional[str] = None,
      example_ids_count: int = flip_count.DEFAULT_NUM_EXAMPLE_IDS,
      name: str = FLIP_RATE_NAME,
      thresholds: Sequence[float] = flip_count.DEFAULT_THRESHOLDS,
      model_name: str = "",
      output_name: str = "",
      eval_config: Optional[config_pb2.EvalConfig] = None,
      example_weighted: bool = False,
  ) -> metric_types.MetricComputations:
      """Returns computations for flip rate."""
      (
          keys,
          metric_key_by_name_by_threshold,
      ) = flip_count.create_metric_keys(
          thresholds,
          _METRICS_LIST,
          name,
          model_name,
          output_name,
          example_weighted,
      )
  
      computations = flip_count.flip_count(
          thresholds=thresholds,
          counterfactual_prediction_key=counterfactual_prediction_key,
          example_id_key=example_id_key,
          example_ids_count=example_ids_count,
          model_name=model_name,
          output_name=output_name,
          eval_config=eval_config,
          example_weighted=example_weighted,
      )
  
      (
          _,
          flip_count_metric_key_by_name_by_threshold,
      ) = flip_count.create_metric_keys(
          thresholds,
          flip_count.METRICS_LIST,
          flip_count.FLIP_COUNT_NAME,
          model_name,
          output_name,
          example_weighted,
      )
  
      def pick_overall_flip_examples(
          ntp_examples: np.ndarray,
          ptn_examples: np.ndarray,
          ntp_count: int,
          ptn_count: int,
      ) -> np.ndarray:
          """Combine samples of flips in both sides to make samples of overall flips.
  
          Note that what this function does is different of simply combining the
          samples and uniformly picking from it. Imagine there are 1M pos-to-neg flips
          and 1k neg-to-pos flips. An correct uniform sampling of overall flips would
          almost certainly contain only pos-to-neg flips, since they are
          overwhelmingly more common.
  
          Args:
            ntp_examples: Pos-to-neg examples.
            ptn_examples: Neg-to-pos examples.
            ntp_count: Total number of neg-to-pos flips.
            ptn_count: Total number of pos-to-neg flips.
  
          Returns:
            array with samples.
          """
          total_flips = ntp_count + ptn_count
          output = []
          ntp_examples = ntp_examples.tolist()
          ptn_examples = ptn_examples.tolist()
          assert len(ntp_examples) <= ntp_count
          assert len(ptn_examples) <= ptn_count
          output_size = min(
              example_ids_count, len(ntp_examples) + len(ptn_examples)
          )
          for _ in range(0, output_size):
              if random.randrange(total_flips) < ntp_count:
                  element_to_remove = random.randrange(
                      len(ntp_examples)
                  )
                  output.append(ntp_examples[element_to_remove])
                  del ntp_examples[element_to_remove]
                  ntp_count = ntp_count - 1
              else:
                  element_to_remove = random.randrange(
                      len(ptn_examples)
                  )
                  output.append(ptn_examples[element_to_remove])
                  del ptn_examples[element_to_remove]
                  ptn_count = ptn_count - 1
              total_flips = total_flips - 1
  
          return np.array(output)
  
      def result(
          metrics: Dict[
              metric_types.MetricKey, Union[float, np.ndarray]
          ]
      ) -> Dict[metric_types.MetricKey, Union[float, np.ndarray]]:
          """Returns flip rate metrics values."""
          output = {}
          for threshold in thresholds:
              ptn = flip_count_metric_key_by_name_by_threshold[
                  threshold
              ][_POSITIVE_TO_NEGATIVE]
              ntp = flip_count_metric_key_by_name_by_threshold[
                  threshold
              ][_NEGATIVE_TO_POSITIVE]
              pos_examples = flip_count_metric_key_by_name_by_threshold[
                  threshold
              ][_POSITIVE_TO_NEGATIVE_EXAMPLE_IDS]
              neg_examples = flip_count_metric_key_by_name_by_threshold[
                  threshold
              ][_NEGATIVE_TO_POSITIVE_EXAMPLE_IDS]
              pos = flip_count_metric_key_by_name_by_threshold[
                  threshold
              ][_POSITIVE_EXAMPLES_COUNT]
              neg = flip_count_metric_key_by_name_by_threshold[
                  threshold
              ][_NEGATIVE_EXAMPLES_COUNT]
              output[
                  metric_key_by_name_by_threshold[threshold][_OVERALL]
              ] = (metrics[ntp] + metrics[ptn]) / (
                  (metrics[pos] + metrics[neg]) or float("NaN")
              )
              output[
                  metric_key_by_name_by_threshold[threshold][
                      _POSITIVE_TO_NEGATIVE
                  ]
              ] = metrics[ptn] / (
                  (metrics[pos] + metrics[neg]) or float("NaN")
              )
              output[
                  metric_key_by_name_by_threshold[threshold][
                      _NEGATIVE_TO_POSITIVE
                  ]
              ] = metrics[ntp] / (
                  (metrics[pos] + metrics[neg]) or float("NaN")
              )
              output[
                  metric_key_by_name_by_threshold[threshold][
                      _POSITIVE_TO_NEGATIVE_EXAMPLE_IDS
                  ]
              ] = metrics[pos_examples]
              output[
                  metric_key_by_name_by_threshold[threshold][
                      _NEGATIVE_TO_POSITIVE_EXAMPLE_IDS
                  ]
              ] = metrics[neg_examples]
              # TODO(sokeefe): Should this depend on  of example_weighted?
              if not example_weighted:
                  assert isinstance(metrics[neg_examples], np.ndarray)
                  assert isinstance(metrics[pos_examples], np.ndarray)
                  output[
                      metric_key_by_name_by_threshold[threshold][
                          _SAMPLE_EXAMPLES_IDS
                      ]
                  ] = pick_overall_flip_examples(
                      ntp_examples=metrics[neg_examples],
                      ptn_examples=metrics[pos_examples],
                      ntp_count=int(metrics[ntp]),
                      ptn_count=int(metrics[ptn]),
                  )
  
          return output
  
      derived_computation = metric_types.DerivedMetricComputation(
          keys=keys, result=result
      )
  
      computations.append(derived_computation)
      return computations
  
file_path: model-analysis/tensorflow_model_analysis/addons/fairness/metrics/counterfactual_fairness/flip_rate.py
imports: import random
  from typing import Dict, Optional, Sequence, Union
  import numpy
  from tensorflow_model_analysis.addons.fairness.metrics.counterfactual_fairness import flip_count
  from tensorflow_model_analysis.metrics import metric_types
  from tensorflow_model_analysis.metrics import metric_util
  from tensorflow_model_analysis.proto import config_pb2
label_desc: Use image node_id 1 for calling a global function with example usage: _flip_rate(counterfactual_prediction_key, example_id_key, example_ids_count, name, thresholds, model_name, output_name, eval_config, example_weighted) and returns: computations, np, output