'Evaluates the predictor
Parameters
recommender: The BaseRecommender instance
The recommender instance to be evaluated.
metric: [None|\'rmse\'|\'f1score\'|\'precision\'|\'recall\'|\'nmae\'|\'mae\']
If metrics is None, all metrics available will be evaluated.
Otherwise it will return the specified metric evaluated.
sampling_users:  float or sampling, optional, default = None
If an float is passed, it is the percentage of evaluated
users. If sampling_users is None, all users are used in the
evaluation. Specific sampling objects can be passed, see
scikits.crab.metrics.sampling module for the list of possible
objects.
sampling_ratings:  float or sampling, optional, default = None
If an float is passed, it is the percentage of evaluated
ratings. If sampling_ratings is None, 70% will be used in the
training set and 30% in the test set. Specific sampling objects
can be passed, see scikits.crab.metrics.sampling module
for the list of possible objects.
at: integer, optional, default = None
This number at is the \'at\' value, as in \'precision at 5\'.  For
example this would mean precision or recall evaluated by removing
the top 5 preferences for a user and then finding the percentage of
those 5 items included in the top 5 recommendations for that user.
If at is None, it will consider all the top 3 elements.
Returns
Returns a dictionary containing the evaluation results:
(NMAE, MAE, RMSE, Precision, Recall, F1-Score)'

def evaluate(self, recommender, metric=None, **kwargs):

    sampling_users = kwargs.pop('sampling_users', None)
    sampling_ratings = kwargs.pop('sampling_ratings', 0.7)
    permutation = kwargs.pop('permutation', True)
    at = kwargs.pop('at', 3)
    if ((metric not in evaluation_metrics) and (metric is not None)):
        raise ValueError(('metric    %s    is    not    recognized.    valid    keywords                                                            are    %s' % (metric, evaluation_metrics.keys())))
    n_users = recommender.model.users_count()
    sampling_users = check_sampling(sampling_users, n_users)
    (users_set, _) = sampling_users.split(permutation=permutation)
    training_set = {}
    testing_set = {}
    user_ids = recommender.model.user_ids()
    for user_id in user_ids[users_set]:
        preferences = recommender.model.preferences_from_user(user_id)
        sampling_eval = check_sampling(sampling_ratings, len(preferences))
        (train_set, test_set) = sampling_eval.split(indices=True, permutation=permutation)
        preferences = list(preferences)
        if recommender.model.has_preference_values():
            training_set[user_id] = (dict((preferences[idx] for idx in train_set)) if preferences else {})
            testing_set[user_id] = ([preferences[idx] for idx in test_set] if preferences else [])
        else:
            training_set[user_id] = (dict(((preferences[idx], 1.0) for idx in train_set)) if preferences else {})
            testing_set[user_id] = ([(preferences[idx], 1.0) for idx in test_set] if preferences else [])
    recommender_training = self._build_recommender(training_set, recommender)
    real_preferences = []
    estimated_preferences = []
    for (user_id, preferences) in testing_set.iteritems():
        for (item_id, preference) in preferences:
            try:
                estimated = recommender_training.estimate_preference(user_id, item_id)
                real_preferences.append(preference)
            except ItemNotFoundError:
                continue
            estimated_preferences.append(estimated)
    if (metric in ['rmse', 'mae', 'nmae']):
        eval_function = evaluation_metrics[metric]
        if (metric == 'nmae'):
            return {metric: eval_function(real_preferences, estimated_preferences, recommender.model.maximum_preference_value(), recommender.model.minimum_preference_value())}
        return {metric: eval_function(real_preferences, estimated_preferences)}
    relevant_arrays = []
    real_arrays = []
    user_ids = recommender.model.user_ids()
    for user_id in user_ids[users_set]:
        preferences = recommender.model.preferences_from_user(user_id)
        preferences = list(preferences)
        if (len(preferences) < (2 * at)):
            continue
        if (not recommender.model.has_preference_values()):
            preferences = [(preference, 1.0) for preference in preferences]
        preferences = sorted(preferences, key=(lambda x: x[1]), reverse=True)
        relevant_item_ids = [item_id for (item_id, preference) in preferences[:at]]
        if (len(relevant_item_ids) == 0):
            continue
        training_set = {}
        for other_user_id in recommender.model.user_ids():
            preferences_other_user = recommender.model.preferences_from_user(other_user_id)
            if (not recommender.model.has_preference_values()):
                preferences_other_user = [(preference, 1.0) for preference in preferences_other_user]
            if (other_user_id == user_id):
                preferences_other_user = [pref for pref in preferences_other_user if (pref[0] not in relevant_item_ids)]
                if preferences_other_user:
                    training_set[other_user_id] = dict(preferences_other_user)
            else:
                training_set[other_user_id] = dict(preferences_other_user)
        recommender_training = self._build_recommender(training_set, recommender)
        try:
            preferences = recommender_training.model.preferences_from_user(user_id)
            preferences = list(preferences)
            if (not preferences):
                continue
        except:
            continue
        recommended_items = recommender_training.recommend(user_id, at)
        relevant_arrays.append(list(relevant_item_ids))
        real_arrays.append(list(recommended_items))
    relevant_arrays = np.array(relevant_arrays)
    real_arrays = np.array(real_arrays)
    if (metric in ['precision', 'recall', 'f1score']):
        eval_function = evaluation_metrics[metric]
        return {metric: eval_function(real_arrays, relevant_arrays)}
    if (metric is None):
        (mae, nmae, rmse) = evaluation_error(real_preferences, estimated_preferences, recommender.model.maximum_preference_value(), recommender.model.minimum_preference_value())
        f = f1_score(real_arrays, relevant_arrays)
        r = recall_score(real_arrays, relevant_arrays)
        p = precision_score(real_arrays, relevant_arrays)
        return {'mae': mae, 'nmae': nmae, 'rmse': rmse, 'precision': p, 'recall': r, 'f1score': f}
