text: import os
  import sys
  
  sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
  
  from cakechat.utils.env import init_cuda_env
  
  init_cuda_env()
  
  from cakechat.utils.dataset_loader import load_datasets
  from cakechat.utils.data_types import Dataset
  from cakechat.utils.logger import get_tools_logger
  from cakechat.dialog_model.factory import get_trained_model
  from cakechat.dialog_model.model_utils import transform_token_ids_to_sentences
  from cakechat.dialog_model.inference import get_nn_responses
  from cakechat.dialog_model.quality import calculate_model_mean_perplexity, get_tfidf_vectorizer, \
      calculate_lexical_similarity
  from cakechat.config import PREDICTION_MODE_FOR_TESTS, DEFAULT_CONDITION
  
  _logger = get_tools_logger(__file__)
  
  
  def _make_non_conditioned(dataset):
      return Dataset(x=dataset.x, y=dataset.y, condition_ids=None)
  
  
  def _slice_condition_data(dataset, condition_id):
      condition_mask = (dataset.condition_ids == condition_id)
      return Dataset(
          x=dataset.x[condition_mask], y=dataset.y[condition_mask], condition_ids=dataset.condition_ids[condition_mask])
  
  
  def calc_perplexity_metrics(nn_model, eval_datasets):
      return {
          'ppl_cs_test':
              calculate_model_mean_perplexity(nn_model, eval_datasets.cs_test),
          'ppl_cs_test_not_conditioned':
              calculate_model_mean_perplexity(nn_model, _make_non_conditioned(eval_datasets.cs_test)),
          'ppl_cs_test_one_condition':
              calculate_model_mean_perplexity(nn_model, eval_datasets.cs_test_one_condition),
          'ppl_cs_test_one_condition_not_conditioned':
              calculate_model_mean_perplexity(nn_model, _make_non_conditioned(eval_datasets.cs_test_one_condition)),
          'ppl_cf_validation':
              calculate_model_mean_perplexity(nn_model, eval_datasets.cf_validation)
      }
  
  
  def calc_perplexity_for_conditions(nn_model, dataset):
      cond_to_ppl_conditioned, cond_to_ppl_not_conditioned = {}, {}
  
      for condition, condition_id in nn_model.condition_to_index.items():
          if condition == DEFAULT_CONDITION:
              continue
  
          dataset_with_conditions = _slice_condition_data(dataset, condition_id)
  
          if not dataset_with_conditions.x.size:
              _logger.warning(
                  'No dataset samples found with the given condition "{}", skipping metrics.'.format(condition))
              continue
  
          cond_to_ppl_conditioned[condition] = \
              calculate_model_mean_perplexity(nn_model, _make_non_conditioned(dataset_with_conditions))
  
          cond_to_ppl_not_conditioned[condition] = \
              calculate_model_mean_perplexity(nn_model, dataset_with_conditions)
  
      return cond_to_ppl_conditioned, cond_to_ppl_not_conditioned
  
  
  def predict_for_condition_id(nn_model, x_val, condition_id=None):
      responses = get_nn_responses(x_val, nn_model, mode=PREDICTION_MODE_FOR_TESTS, condition_ids=condition_id)
      return [candidates[0] for candidates in responses]
  
  
  def calc_lexical_similarity_metrics(nn_model, testset, tfidf_vectorizer):
      """
      For each condition calculate lexical similarity between ground-truth responses and
      generated conditioned responses. Similarly compare ground-truth responses with non-conditioned generated responses.
      If lex_sim(gt, cond_resp) > lex_sim(gt, non_cond_resp), the conditioning on extra information proves to be useful.
      :param nn_model: trained model to evaluate
      :param testset: context-sensitive testset, instance of Dataset
      :param tfidf_vectorizer: instance of scikit-learn TfidfVectorizer, calculates lexical similariry for documents
      according to TF-IDF metric
      :return: two dictionaries:
          {condition: lex_sim(gt, cond_resp)},
          {condition: lex_sim(gt, non_cond_resp)}
      """
      gt_vs_cond_lex_sim, gt_vs_non_cond_lex_sim = {}, {}
  
      for condition, condition_id in nn_model.condition_to_index.items():
          sample_mask_for_condition = testset.condition_ids == condition_id
          contexts_for_condition = testset.x[sample_mask_for_condition]
          responses_for_condition = testset.y[sample_mask_for_condition]
  
          if not responses_for_condition.size:
              _logger.warning('No dataset samples found for condition "{}", skip it.'.format(condition))
              continue
  
          gt_responses = transform_token_ids_to_sentences(responses_for_condition, nn_model.index_to_token)
          conditioned_responses = predict_for_condition_id(nn_model, contexts_for_condition, condition_id)
          non_conditioned_responses = predict_for_condition_id(nn_model, contexts_for_condition, condition_id=None)
  
          gt_vs_cond_lex_sim[condition] = \
              calculate_lexical_similarity(gt_responses, conditioned_responses, tfidf_vectorizer)
  
          gt_vs_non_cond_lex_sim[condition] = \
              calculate_lexical_similarity(gt_responses, non_conditioned_responses, tfidf_vectorizer)
  
      return gt_vs_cond_lex_sim, gt_vs_non_cond_lex_sim
  
  
  if __name__ == '__main__':
      nn_model = get_trained_model()
      eval_datasets = load_datasets(nn_model.token_to_index, nn_model.condition_to_index)
  
      print('\nPerplexity on datasets:')
      for dataset, perplexity in calc_perplexity_metrics(nn_model, eval_datasets).items():
          print('\t{}: \t{:.1f}'.format(dataset, perplexity))
  
      cond_to_ppl_conditioned, cond_to_ppl_not_conditioned = \
          calc_perplexity_for_conditions(nn_model, eval_datasets.cs_test)
  
      print('\nPerplexity on conditioned testset for conditions:')
      for condition, perplexity in cond_to_ppl_conditioned.items():
          print('\t{}: \t{:.1f}'.format(condition, perplexity))
  
      print('\nPerplexity on non-conditioned testset for conditions:')
      for condition, perplexity in cond_to_ppl_not_conditioned.items():
          print('\t{}: \t{:.1f}'.format(condition, perplexity))
  
      gt_vs_cond_lex_sim, gt_vs_non_cond_lex_sim = \
          calc_lexical_similarity_metrics(nn_model, eval_datasets.cs_test, get_tfidf_vectorizer())
  
      print('\nLexical similarity, ground-truth vs. conditioned responses:')
      for condition, lex_sim in gt_vs_cond_lex_sim.items():
          print('\t{}: \t{:.2f}'.format(condition, lex_sim))
  
      print('\nLexical similarity, ground-truth vs. non-conditioned responses:')
      for condition, lex_sim in gt_vs_non_cond_lex_sim.items():
          print('\t{}: \t{:.2f}'.format(condition, lex_sim))
  