rasa/core/test.py
async def compare(models: Text,
                  stories_file: Text,
                  output: Text) -> None:
    """Evaluates multiple trained models on a test set."""
    from rasa.core.agent import Agent
    import rasa_nlu.utils as nlu_utils
    from rasa.core import utils

    num_correct = defaultdict(list)

    for run in nlu_utils.list_subdirectories(models):
        num_correct_run = defaultdict(list)

        for model in sorted(nlu_utils.list_subdirectories(run)):
            logger.info("Evaluating model {}".format(model))

            agent = Agent.load(model)

            completed_trackers = await _generate_trackers(stories_file, agent)

            story_eval_store, no_of_stories = \
                collect_story_predictions(completed_trackers,
                                          agent)

            failed_stories = story_eval_store.failed_stories
            policy_name = ''.join(
                [i for i in os.path.basename(model) if not i.isdigit()])
            num_correct_run[policy_name].append(no_of_stories -
                                                len(failed_stories))

        for k, v in num_correct_run.items():
            num_correct[k].append(v)

    utils.dump_obj_as_json_to_file(os.path.join(output, 'results.json'),
                                   num_correct)
Evaluates multiple trained models on a test set.
Evaluates multiple trained models on a test set.
Evaluates multiple trained models on a test set .
async def compare(models: Text,
                  stories_file: Text,
                  output: Text) -> None:
    from rasa.core.agent import Agent
    import rasa_nlu.utils as nlu_utils
    from rasa.core import utils
    num_correct = defaultdict(list)
    for run in nlu_utils.list_subdirectories(models):
        num_correct_run = defaultdict(list)
        for model in sorted(nlu_utils.list_subdirectories(run)):
            logger.info("Evaluating model {}".format(model))
            agent = Agent.load(model)
            completed_trackers = await _generate_trackers(stories_file, agent)
            story_eval_store, no_of_stories = \
                collect_story_predictions(completed_trackers,
                                          agent)
            failed_stories = story_eval_store.failed_stories
            policy_name = ''.join(
                [i for i in os.path.basename(model) if not i.isdigit()])
            num_correct_run[policy_name].append(no_of_stories -
                                                len(failed_stories))
        for k, v in num_correct_run.items():
            num_correct[k].append(v)
    utils.dump_obj_as_json_to_file(os.path.join(output, 'results.json'),
                                   num_correct)
async def compare ( models : Text , stories_file : Text , output : Text ) -> None : from rasa . core . agent import Agent import rasa_nlu . utils as nlu_utils from rasa . core import utils num_correct = defaultdict ( list ) for run in nlu_utils . list_subdirectories ( models ) : num_correct_run = defaultdict ( list ) for model in sorted ( nlu_utils . list_subdirectories ( run ) ) : logger . info ( "Evaluating model {}" . format ( model ) ) agent = Agent . load ( model ) completed_trackers = await _generate_trackers ( stories_file , agent ) story_eval_store , no_of_stories = collect_story_predictions ( completed_trackers , agent ) failed_stories = story_eval_store . failed_stories policy_name = '' . join ( [ i for i in os . path . basename ( model ) if not i . isdigit ( ) ] ) num_correct_run [ policy_name ] . append ( no_of_stories - len ( failed_stories ) ) for k , v in num_correct_run . items ( ) : num_correct [ k ] . append ( v ) utils . dump_obj_as_json_to_file ( os . path . join ( output , 'results.json' ) , num_correct )
Evaluates multiple trained models on a test set.
Evaluates multiple trained models on a test set .
"""Evaluates multiple trained models on a test set."""
models
Text
stories_file
Text
output
Text