@beam.ptransform_fn
@beam.typehints.with_output_types(beam.pvalue.PDone)
def ExtractEvaluateAndWriteResults(examples: beam.pvalue.PCollection, eval_shared_model: Optional[Union[(types.EvalSharedModel, Dict[(Text, types.EvalSharedModel)])]]=None, eval_config: config.EvalConfig=None, extractors: Optional[List[extractor.Extractor]]=None, evaluators: Optional[List[evaluator.Evaluator]]=None, writers: Optional[List[writer.Writer]]=None, output_path: Optional[Text]=None, display_only_data_location: Optional[Text]=None, display_only_file_format: Optional[Text]=None, slice_spec: Optional[List[slicer.SingleSliceSpec]]=None, write_config: Optional[bool]=True, compute_confidence_intervals: Optional[bool]=False, k_anonymization_count: int=1, desired_batch_size: Optional[int]=None, random_seed_for_testing: Optional[int]=None) -> beam.pvalue.PDone:
    "PTransform for performing extraction, evaluation, and writing results.\n\n  Users who want to construct their own Beam pipelines instead of using the\n  lightweight run_model_analysis functions should use this PTransform.\n\n  Example usage:\n    eval_config = tfma.EvalConfig(slicing_specs=[...], metrics_specs=[...])\n    eval_shared_model = tfma.default_eval_shared_model(\n        eval_saved_model_path=model_location, eval_config=eval_config)\n    with beam.Pipeline(runner=...) as p:\n      _ = (p\n           | 'ReadData' >> beam.io.ReadFromTFRecord(data_location)\n           | 'ExtractEvaluateAndWriteResults' >>\n           tfma.ExtractEvaluateAndWriteResults(\n               eval_shared_model=eval_shared_model,\n               eval_config=eval_config,\n               ...))\n    result = tfma.load_eval_result(output_path=output_path)\n    tfma.view.render_slicing_metrics(result)\n\n  Note that the exact serialization format is an internal implementation detail\n  and subject to change. Users should only use the TFMA functions to write and\n  read the results.\n\n  Args:\n    examples: PCollection of input examples. Can be any format the model accepts\n      (e.g. string containing CSV row, TensorFlow.Example, etc).\n    eval_shared_model: Optional shared model (single-model evaluation) or dict\n      of shared models keyed by model name (multi-model evaluation). Only\n      required if needed by default extractors, evaluators, or writers and for\n      display purposes of the model path.\n    eval_config: Eval config.\n    extractors: Optional list of Extractors to apply to Extracts. Typically\n      these will be added by calling the default_extractors function. If no\n      extractors are provided, default_extractors (non-materialized) will be\n      used.\n    evaluators: Optional list of Evaluators for evaluating Extracts. Typically\n      these will be added by calling the default_evaluators function. If no\n      evaluators are provided, default_evaluators will be used.\n    writers: Optional list of Writers for writing Evaluation output. Typically\n      these will be added by calling the default_writers function. If no writers\n      are provided, default_writers will be used.\n    output_path: Path to output metrics and plots results.\n    display_only_data_location: Optional path indicating where the examples were\n      read from. This is used only for display purposes - data will not actually\n      be read from this path.\n    display_only_file_format: Optional format of the examples. This is used only\n      for display purposes.\n    slice_spec: Deprecated (use EvalConfig).\n    write_config: Deprecated (use EvalConfig).\n    compute_confidence_intervals: Deprecated (use EvalConfig).\n    k_anonymization_count: Deprecated (use EvalConfig).\n    desired_batch_size: Optional batch size for batching in Predict.\n    random_seed_for_testing: Provide for deterministic tests only.\n\n  Raises:\n    ValueError: If EvalConfig invalid or matching Extractor not found for an\n      Evaluator.\n\n  Returns:\n    PDone.\n  "
    eval_shared_models = eval_shared_model
    if (not isinstance(eval_shared_model, dict)):
        eval_shared_models = {'': eval_shared_model}
    if (eval_config is None):
        model_specs = []
        for (model_name, shared_model) in eval_shared_models.items():
            example_weight_key = shared_model.example_weight_key
            example_weight_keys = {}
            if (example_weight_key and isinstance(example_weight_key, dict)):
                example_weight_keys = example_weight_key
                example_weight_key = ''
            model_specs.append(config.ModelSpec(name=model_name, example_weight_key=example_weight_key, example_weight_keys=example_weight_keys))
        slicing_specs = None
        if slice_spec:
            slicing_specs = [s.to_proto() for s in slice_spec]
        options = config.Options()
        options.compute_confidence_intervals.value = compute_confidence_intervals
        options.k_anonymization_count.value = k_anonymization_count
        if (not write_config):
            options.disabled_outputs.values.append(_EVAL_CONFIG_FILE)
        eval_config = config.EvalConfig(model_specs=model_specs, slicing_specs=slicing_specs, options=options)
    else:
        eval_config = config.update_eval_config_with_defaults(eval_config)
    config.verify_eval_config(eval_config)
    if (not extractors):
        extractors = default_extractors(eval_config=eval_config, eval_shared_model=eval_shared_model, materialize=False, desired_batch_size=desired_batch_size)
    if (not evaluators):
        evaluators = default_evaluators(eval_config=eval_config, eval_shared_model=eval_shared_model, random_seed_for_testing=random_seed_for_testing)
    for v in evaluators:
        evaluator.verify_evaluator(v, extractors)
    if (not writers):
        writers = default_writers(output_path=output_path, eval_shared_model=eval_shared_model)
    _ = (((examples | ('InputsToExtracts' >> InputsToExtracts())) | ('ExtractAndEvaluate' >> ExtractAndEvaluate(extractors=extractors, evaluators=evaluators))) | ('WriteResults' >> WriteResults(writers=writers)))
    if (_EVAL_CONFIG_FILE not in eval_config.options.disabled_outputs.values):
        data_location = '<user provided PCollection>'
        if (display_only_data_location is not None):
            data_location = display_only_data_location
        file_format = '<unknown>'
        if (display_only_file_format is not None):
            file_format = display_only_file_format
        model_locations = {}
        for (k, v) in eval_shared_models.items():
            model_locations[k] = ('<unknown>' if ((v is None) or (v.model_path is None)) else v.model_path)
        _ = (examples.pipeline | WriteEvalConfig(eval_config, output_path, data_location, file_format, model_locations))
    return beam.pvalue.PDone(examples.pipeline)
PTransform for performing extraction, evaluation, and writing results.

Users who want to construct their own Beam pipelines instead of using the
lightweight run_model_analysis functions should use this PTransform.

Example usage:
  eval_config = tfma.EvalConfig(slicing_specs=[...], metrics_specs=[...])
  eval_shared_model = tfma.default_eval_shared_model(
      eval_saved_model_path=model_location, eval_config=eval_config)
  with beam.Pipeline(runner=...) as p:
    _ = (p
         | 'ReadData' >> beam.io.ReadFromTFRecord(data_location)
         | 'ExtractEvaluateAndWriteResults' >>
         tfma.ExtractEvaluateAndWriteResults(
             eval_shared_model=eval_shared_model,
             eval_config=eval_config,
             ...))
  result = tfma.load_eval_result(output_path=output_path)
  tfma.view.render_slicing_metrics(result)

Note that the exact serialization format is an internal implementation detail
and subject to change. Users should only use the TFMA functions to write and
read the results.

Args:
  examples: PCollection of input examples. Can be any format the model accepts
    (e.g. string containing CSV row, TensorFlow.Example, etc).
  eval_shared_model: Optional shared model (single-model evaluation) or dict
    of shared models keyed by model name (multi-model evaluation). Only
    required if needed by default extractors, evaluators, or writers and for
    display purposes of the model path.
  eval_config: Eval config.
  extractors: Optional list of Extractors to apply to Extracts. Typically
    these will be added by calling the default_extractors function. If no
    extractors are provided, default_extractors (non-materialized) will be
    used.
  evaluators: Optional list of Evaluators for evaluating Extracts. Typically
    these will be added by calling the default_evaluators function. If no
    evaluators are provided, default_evaluators will be used.
  writers: Optional list of Writers for writing Evaluation output. Typically
    these will be added by calling the default_writers function. If no writers
    are provided, default_writers will be used.
  output_path: Path to output metrics and plots results.
  display_only_data_location: Optional path indicating where the examples were
    read from. This is used only for display purposes - data will not actually
    be read from this path.
  display_only_file_format: Optional format of the examples. This is used only
    for display purposes.
  slice_spec: Deprecated (use EvalConfig).
  write_config: Deprecated (use EvalConfig).
  compute_confidence_intervals: Deprecated (use EvalConfig).
  k_anonymization_count: Deprecated (use EvalConfig).
  desired_batch_size: Optional batch size for batching in Predict.
  random_seed_for_testing: Provide for deterministic tests only.

Raises:
  ValueError: If EvalConfig invalid or matching Extractor not found for an
    Evaluator.

Returns:
  PDone.
@beam.ptransform_fn
@beam.typehints.with_output_types(beam.pvalue.PDone)
def ExtractEvaluateAndWriteResults(examples: beam.pvalue.PCollection, eval_shared_model: Optional[Union[(types.EvalSharedModel, Dict[(Text, types.EvalSharedModel)])]]=None, eval_config: config.EvalConfig=None, extractors: Optional[List[extractor.Extractor]]=None, evaluators: Optional[List[evaluator.Evaluator]]=None, writers: Optional[List[writer.Writer]]=None, output_path: Optional[Text]=None, display_only_data_location: Optional[Text]=None, display_only_file_format: Optional[Text]=None, slice_spec: Optional[List[slicer.SingleSliceSpec]]=None, write_config: Optional[bool]=True, compute_confidence_intervals: Optional[bool]=False, k_anonymization_count: int=1, desired_batch_size: Optional[int]=None, random_seed_for_testing: Optional[int]=None) -> beam.pvalue.PDone:
    "PTransform for performing extraction, evaluation, and writing results.\n\n  Users who want to construct their own Beam pipelines instead of using the\n  lightweight run_model_analysis functions should use this PTransform.\n\n  Example usage:\n    eval_config = tfma.EvalConfig(slicing_specs=[...], metrics_specs=[...])\n    eval_shared_model = tfma.default_eval_shared_model(\n        eval_saved_model_path=model_location, eval_config=eval_config)\n    with beam.Pipeline(runner=...) as p:\n      _ = (p\n           | 'ReadData' >> beam.io.ReadFromTFRecord(data_location)\n           | 'ExtractEvaluateAndWriteResults' >>\n           tfma.ExtractEvaluateAndWriteResults(\n               eval_shared_model=eval_shared_model,\n               eval_config=eval_config,\n               ...))\n    result = tfma.load_eval_result(output_path=output_path)\n    tfma.view.render_slicing_metrics(result)\n\n  Note that the exact serialization format is an internal implementation detail\n  and subject to change. Users should only use the TFMA functions to write and\n  read the results.\n\n  Args:\n    examples: PCollection of input examples. Can be any format the model accepts\n      (e.g. string containing CSV row, TensorFlow.Example, etc).\n    eval_shared_model: Optional shared model (single-model evaluation) or dict\n      of shared models keyed by model name (multi-model evaluation). Only\n      required if needed by default extractors, evaluators, or writers and for\n      display purposes of the model path.\n    eval_config: Eval config.\n    extractors: Optional list of Extractors to apply to Extracts. Typically\n      these will be added by calling the default_extractors function. If no\n      extractors are provided, default_extractors (non-materialized) will be\n      used.\n    evaluators: Optional list of Evaluators for evaluating Extracts. Typically\n      these will be added by calling the default_evaluators function. If no\n      evaluators are provided, default_evaluators will be used.\n    writers: Optional list of Writers for writing Evaluation output. Typically\n      these will be added by calling the default_writers function. If no writers\n      are provided, default_writers will be used.\n    output_path: Path to output metrics and plots results.\n    display_only_data_location: Optional path indicating where the examples were\n      read from. This is used only for display purposes - data will not actually\n      be read from this path.\n    display_only_file_format: Optional format of the examples. This is used only\n      for display purposes.\n    slice_spec: Deprecated (use EvalConfig).\n    write_config: Deprecated (use EvalConfig).\n    compute_confidence_intervals: Deprecated (use EvalConfig).\n    k_anonymization_count: Deprecated (use EvalConfig).\n    desired_batch_size: Optional batch size for batching in Predict.\n    random_seed_for_testing: Provide for deterministic tests only.\n\n  Raises:\n    ValueError: If EvalConfig invalid or matching Extractor not found for an\n      Evaluator.\n\n  Returns:\n    PDone.\n  "
    eval_shared_models = eval_shared_model
    if (not isinstance(eval_shared_model, dict)):
        eval_shared_models = {: eval_shared_model}
    if (eval_config is None):
        model_specs = []
        for (model_name, shared_model) in eval_shared_models.items():
            example_weight_key = shared_model.example_weight_key
            example_weight_keys = {}
            if (example_weight_key and isinstance(example_weight_key, dict)):
                example_weight_keys = example_weight_key
                example_weight_key = 
            model_specs.append(config.ModelSpec(name=model_name, example_weight_key=example_weight_key, example_weight_keys=example_weight_keys))
        slicing_specs = None
        if slice_spec:
            slicing_specs = [s.to_proto() for s in slice_spec]
        options = config.Options()
        options.compute_confidence_intervals.value = compute_confidence_intervals
        options.k_anonymization_count.value = k_anonymization_count
        if (not write_config):
            options.disabled_outputs.values.append(_EVAL_CONFIG_FILE)
        eval_config = config.EvalConfig(model_specs=model_specs, slicing_specs=slicing_specs, options=options)
    else:
        eval_config = config.update_eval_config_with_defaults(eval_config)
    config.verify_eval_config(eval_config)
    if (not extractors):
        extractors = default_extractors(eval_config=eval_config, eval_shared_model=eval_shared_model, materialize=False, desired_batch_size=desired_batch_size)
    if (not evaluators):
        evaluators = default_evaluators(eval_config=eval_config, eval_shared_model=eval_shared_model, random_seed_for_testing=random_seed_for_testing)
    for v in evaluators:
        evaluator.verify_evaluator(v, extractors)
    if (not writers):
        writers = default_writers(output_path=output_path, eval_shared_model=eval_shared_model)
    _ = (((examples | ('InputsToExtracts' >> InputsToExtracts())) | ('ExtractAndEvaluate' >> ExtractAndEvaluate(extractors=extractors, evaluators=evaluators))) | ('WriteResults' >> WriteResults(writers=writers)))
    if (_EVAL_CONFIG_FILE not in eval_config.options.disabled_outputs.values):
        data_location = '<user provided PCollection>'
        if (display_only_data_location is not None):
            data_location = display_only_data_location
        file_format = '<unknown>'
        if (display_only_file_format is not None):
            file_format = display_only_file_format
        model_locations = {}
        for (k, v) in eval_shared_models.items():
            model_locations[k] = ('<unknown>' if ((v is None) or (v.model_path is None)) else v.model_path)
        _ = (examples.pipeline | WriteEvalConfig(eval_config, output_path, data_location, file_format, model_locations))
    return beam.pvalue.PDone(examples.pipeline)