class_docstr: Propensity-based Stratified Evaluation Method proposed by Jadidinejad et al. (2021)
  
  Parameters
  ----------
  data: array-like, required
      Raw preference data in the triplet format [(user_id, item_id, rating_value)].
  
  test_size: float, optional, default: 0.2
      The proportion of the test set, 
      if > 1 then it is treated as the size of the test set.
  
  val_size: float, optional, default: 0.0
      The proportion of the validation set,         if > 1 then it is treated as the size of the validation set.
  
  n_strata: int, optional, default: 2
      The number of strata for propensity-based stratification.
  
  rating_threshold: float, optional, default: 1.0
      Threshold used to binarize rating values into positive or negative feedback for
      model evaluation using ranking metrics (rating metrics are not affected).
  
  seed: int, optional, default: None
      Random seed for reproducibility.
  
  exclude_unknowns: bool, optional, default: True
      If `True`, unknown users and items will be ignored during model evaluation.
  
  verbose: bool, optional, default: False
      Output running log.
  
  References
  ----------
  Amir H. Jadidinejad, Craig Macdonald and Iadh Ounis, 
  The Simpson's Paradox in the Offline Evaluation of Recommendation Systems, 
  ACM Transactions on Information Systems (to appear)
  https://arxiv.org/abs/2104.08912
class_docstr_tok: ["Propensity-based","Stratified","Evaluation","Method","proposed","by","Jadidinejad","et","al",".","(","2021",")","Parameters","--","--","--","--","--","data",":","array-like",",","required","Raw","preference","data","in","the","triplet","format","[","(","user_id",",","item_id",",","rating_value",")","]",".","test_size",":","float",",","optional",",","default",":","0.2","The","proportion","of","the","test","set",",","if",">","1","then","it","is","treated","as","the","size","of","the","test","set",".","val_size",":","float",",","optional",",","default",":","0.0","The","proportion","of","the","validation","set",",","if",">","1","then","it","is","treated","as","the","size","of","the","validation","set",".","n_strata",":","int",",","optional",",","default",":","2","The","number","of","strata","for","propensity-based","stratification",".","rating_threshold",":","float",",","optional",",","default",":","1.0","Threshold","used","to","binarize","rating","values","into","positive","or","negative","feedback","for","model","evaluation","using","ranking","metrics","(","rating","metrics","are","not","affected",")",".","seed",":","int",",","optional",",","default",":","None","Random","seed","for","reproducibility",".","exclude_unknowns",":","bool",",","optional",",","default",":","True","If","`","True","`",",","unknown","users","and","items","will","be","ignored","during","model","evaluation",".","verbose",":","bool",",","optional",",","default",":","False","Output","running","log",".","References","--","--","--","--","--","Amir","H.","Jadidinejad",",","Craig","Macdonald","and","Iadh","Ounis",",","The","Simpson","'s","Paradox","in","the","Offline","Evaluation","of","Recommendation","Systems",",","ACM","Transactions","on","Information","Systems","(","to","appear",")","https",":","\/\/arxiv.org\/abs\/2104.08912"]
docstr: Evaluate given models according to given metrics
  
  Parameters
  ----------
  model: :obj:`cornac.models.Recommender`
      Recommender model to be evaluated.
  
  metrics: :obj:`iterable`
      List of metrics.
  
  user_based: bool, required
      Evaluation strategy for the rating metrics. Whether results
      are averaging based on number of users or number of ratings.
  
  show_validation: bool, optional, default: True
      Whether to show the results on validation set (if exists).
  
  Returns
  -------
  res: :obj:`cornac.experiment.Result`
docstr_tok: ["Evaluate","given","models","according","to","given","metrics","Parameters","--","--","--","--","--","model",":",":","obj",":","`","cornac.models.Recommender","`","Recommender","model","to","be","evaluated",".","metrics",":",":","obj",":","`","iterable","`","List","of","metrics",".","user_based",":","bool",",","required","Evaluation","strategy","for","the","rating","metrics",".","Whether","results","are","averaging","based","on","number","of","users","or","number","of","ratings",".","show_validation",":","bool",",","optional",",","default",":","True","Whether","to","show","the","results","on","validation","set","(","if","exists",")",".","Returns","--","--","--","-","res",":",":","obj",":","`","cornac.experiment.Result","`"]
code: def evaluate(self, model, metrics, user_based, show_validation=True):
      """Evaluate given models according to given metrics
  
      Parameters
      ----------
      model: :obj:`cornac.models.Recommender`
          Recommender model to be evaluated.
  
      metrics: :obj:`iterable`
          List of metrics.
  
      user_based: bool, required
          Evaluation strategy for the rating metrics. Whether results
          are averaging based on number of users or number of ratings.
  
      show_validation: bool, optional, default: True
          Whether to show the results on validation set (if exists).
  
      Returns
      -------
      res: :obj:`cornac.experiment.Result`
      """
      result = PSTResult(model.name)
  
      if self.train_set is None:
          raise ValueError("train_set is required but None!")
      if self.test_set is None:
          raise ValueError("test_set is required but None!")
  
      self._reset()
      self._organize_metrics(metrics)
  
      ###########
      # FITTING #
      ###########
      if self.verbose:
          print("\n[{}] Training started!".format(model.name))
  
      start = time.time()
      model.fit(self.train_set, self.val_set)
      train_time = time.time() - start
  
      ##############
      # EVALUATION #
      ##############
      if self.verbose:
          print("\n[{}] Evaluation started!".format(model.name))
  
      # evaluate on the sampled test set (closed-loop)
      test_result = self._eval(
          model=model,
          test_set=self.test_set,
          val_set=self.val_set,
          user_based=user_based,
      )
      test_result.metric_avg_results["SIZE"] = self.test_set.num_ratings
      result.append(test_result)
  
      if self.verbose:
          print("\n[{}] IPS Evaluation started!".format(model.name))
  
      # evaluate based on Inverse Propensity Scoring
      ips_result = self._eval(
          model=model,
          test_set=self.test_set,
          val_set=self.val_set,
          user_based=user_based,
          props=self.props,
      )
      ips_result.metric_avg_results["SIZE"] = self.test_set.num_ratings
      result.append(ips_result)
  
      if self.verbose:
          print(
              "\n[{}] Stratified Evaluation started!".format(model.name)
          )
  
      # evaluate on different strata
      start = time.time()
  
      for _, qtest_set in self.stratified_sets.items():
          qtest_result = self._eval(
              model=model,
              test_set=qtest_set,
              val_set=self.val_set,
              user_based=user_based,
          )
  
          test_time = time.time() - start
          qtest_result.metric_avg_results[
              "SIZE"
          ] = qtest_set.num_ratings
  
          result.append(qtest_result)
  
      result.organize()
  
      val_result = None
      if show_validation and self.val_set is not None:
          start = time.time()
          val_result = self._eval(
              model=model,
              test_set=self.val_set,
              val_set=None,
              user_based=user_based,
          )
          val_time = time.time() - start
  
      return result, val_result
  
file_path: cornac/cornac/eval_methods/propensity_stratified_evaluation.py
imports: import time
  from collections import defaultdict
  from collections import OrderedDict
  import powerlaw
  import numpy
  import tqdm.auto
  from ..utils.common import safe_indexing
  from ..data import Dataset
  from .base_method import BaseMethod, rating_eval
  from .ratio_split import RatioSplit
  from ..experiment.result import Result, PSTResult
label_desc: Use image node_id 6 for calling the PropensityStratifiedEvaluation obj's underlying member method code with example usage: obj.evaluate(model, metrics, user_based, show_validation) and returns: result,  val_result