rasa/core/test.py
def collect_story_predictions(
    completed_trackers: List['DialogueStateTracker'],
    agent: 'Agent',
    fail_on_prediction_errors: bool = False,
    use_e2e: bool = False
) -> Tuple[StoryEvalution, int]:
    """Test the stories from a file, running them through the stored model."""
    from rasa_nlu.test import get_evaluation_metrics
    from tqdm import tqdm

    story_eval_store = EvaluationStore()
    failed = []
    correct_dialogues = []
    num_stories = len(completed_trackers)

    logger.info("Evaluating {} stories\n"
                "Progress:".format(num_stories))

    action_list = []

    for tracker in tqdm(completed_trackers):
        tracker_results, predicted_tracker, tracker_actions = \
            _predict_tracker_actions(tracker, agent,
                                     fail_on_prediction_errors, use_e2e)

        story_eval_store.merge_store(tracker_results)

        action_list.extend(tracker_actions)

        if tracker_results.has_prediction_target_mismatch():
            # there is at least one wrong prediction
            failed.append(predicted_tracker)
            correct_dialogues.append(0)
        else:
            correct_dialogues.append(1)

    logger.info("Finished collecting predictions.")
    with warnings.catch_warnings():
        from sklearn.exceptions import UndefinedMetricWarning

        warnings.simplefilter("ignore", UndefinedMetricWarning)
        report, precision, f1, accuracy = get_evaluation_metrics(
            [1] * len(completed_trackers), correct_dialogues)

    in_training_data_fraction = _in_training_data_fraction(action_list)

    log_evaluation_table([1] * len(completed_trackers),
                         "END-TO-END" if use_e2e else "CONVERSATION",
                         report, precision, f1, accuracy,
                         in_training_data_fraction,
                         include_report=False)

    return (StoryEvalution(evaluation_store=story_eval_store,
                           failed_stories=failed,
                           action_list=action_list,
                           in_training_data_fraction=in_training_data_fraction),
            num_stories)
Test the stories from a file, running them through the stored model.
Test the stories from a file, running them through the stored model.
Test the stories from a file running them through the stored model .
def collect_story_predictions(
    completed_trackers: List['DialogueStateTracker'],
    agent: 'Agent',
    fail_on_prediction_errors: bool = False,
    use_e2e: bool = False
) -> Tuple[StoryEvalution, int]:
    from rasa_nlu.test import get_evaluation_metrics
    from tqdm import tqdm
    story_eval_store = EvaluationStore()
    failed = []
    correct_dialogues = []
    num_stories = len(completed_trackers)
    logger.info("Evaluating {} stories\n"
                "Progress:".format(num_stories))
    action_list = []
    for tracker in tqdm(completed_trackers):
        tracker_results, predicted_tracker, tracker_actions = \
            _predict_tracker_actions(tracker, agent,
                                     fail_on_prediction_errors, use_e2e)
        story_eval_store.merge_store(tracker_results)
        action_list.extend(tracker_actions)
        if tracker_results.has_prediction_target_mismatch():
            failed.append(predicted_tracker)
            correct_dialogues.append(0)
        else:
            correct_dialogues.append(1)
    logger.info("Finished collecting predictions.")
    with warnings.catch_warnings():
        from sklearn.exceptions import UndefinedMetricWarning
        warnings.simplefilter("ignore", UndefinedMetricWarning)
        report, precision, f1, accuracy = get_evaluation_metrics(
            [1] * len(completed_trackers), correct_dialogues)
    in_training_data_fraction = _in_training_data_fraction(action_list)
    log_evaluation_table([1] * len(completed_trackers),
                         "END-TO-END" if use_e2e else "CONVERSATION",
                         report, precision, f1, accuracy,
                         in_training_data_fraction,
                         include_report=False)
    return (StoryEvalution(evaluation_store=story_eval_store,
                           failed_stories=failed,
                           action_list=action_list,
                           in_training_data_fraction=in_training_data_fraction),
            num_stories)
def collect_story_predictions ( completed_trackers : List [ 'DialogueStateTracker' ] , agent : 'Agent' , fail_on_prediction_errors : bool = False , use_e2e : bool = False ) -> Tuple [ StoryEvalution , int ] : from rasa_nlu . test import get_evaluation_metrics from tqdm import tqdm story_eval_store = EvaluationStore ( ) failed = [ ] correct_dialogues = [ ] num_stories = len ( completed_trackers ) logger . info ( "Evaluating {} stories\n" "Progress:" . format ( num_stories ) ) action_list = [ ] for tracker in tqdm ( completed_trackers ) : tracker_results , predicted_tracker , tracker_actions = _predict_tracker_actions ( tracker , agent , fail_on_prediction_errors , use_e2e ) story_eval_store . merge_store ( tracker_results ) action_list . extend ( tracker_actions ) if tracker_results . has_prediction_target_mismatch ( ) : failed . append ( predicted_tracker ) correct_dialogues . append ( 0 ) else : correct_dialogues . append ( 1 ) logger . info ( "Finished collecting predictions." ) with warnings . catch_warnings ( ) : from sklearn . exceptions import UndefinedMetricWarning warnings . simplefilter ( "ignore" , UndefinedMetricWarning ) report , precision , f1 , accuracy = get_evaluation_metrics ( [ 1 ] * len ( completed_trackers ) , correct_dialogues ) in_training_data_fraction = _in_training_data_fraction ( action_list ) log_evaluation_table ( [ 1 ] * len ( completed_trackers ) , "END-TO-END" if use_e2e else "CONVERSATION" , report , precision , f1 , accuracy , in_training_data_fraction , include_report = False ) return ( StoryEvalution ( evaluation_store = story_eval_store , failed_stories = failed , action_list = action_list , in_training_data_fraction = in_training_data_fraction ) , num_stories )
Test the stories from a file, running them through the stored model.
Test the stories from a file running them through the stored model .
"""Test the stories from a file, running them through the stored model.""" # there is at least one wrong prediction
completed_trackers
List['DialogueStateTracker']
agent
'Agent'
fail_on_prediction_errors
bool
use_e2e
bool