from itertools import product

import numpy as np
import pandas as pd
import pytest
from sklearn.metrics import matthews_corrcoef as sk_matthews_corrcoef

from evalml.objectives import (
    F1,
    MAPE,
    MSE,
    AccuracyBinary,
    AccuracyMulticlass,
    BalancedAccuracyBinary,
    BalancedAccuracyMulticlass,
    BinaryClassificationObjective,
    CostBenefitMatrix,
    ExpVariance,
    F1Macro,
    F1Micro,
    F1Weighted,
    LogLossBinary,
    MCCBinary,
    MCCMulticlass,
    MeanSquaredLogError,
    Precision,
    PrecisionMacro,
    PrecisionMicro,
    PrecisionWeighted,
    Recall,
    RecallMacro,
    RecallMicro,
    RecallWeighted,
    RootMeanSquaredError,
    RootMeanSquaredLogError
)
from evalml.objectives.utils import (
    _all_objectives_dict,
    get_non_core_objectives
)

EPS = 1e-5
all_automl_objectives = _all_objectives_dict()
all_automl_objectives = {name: class_() for name, class_ in all_automl_objectives.items() if class_ not in get_non_core_objectives()}


def test_input_contains_nan():
    y_predicted = np.array([np.nan, 0, 0])
    y_true = np.array([1, 2, 1])
    for objective in all_automl_objectives.values():
        with pytest.raises(ValueError, match="y_predicted contains NaN or infinity"):
            objective.score(y_true, y_predicted)

    y_true = np.array([np.nan, 0, 0])
    y_predicted = np.array([1, 2, 0])
    for objective in all_automl_objectives.values():
        with pytest.raises(ValueError, match="y_true contains NaN or infinity"):
            objective.score(y_true, y_predicted)

    y_true = np.array([1, 0])
    y_predicted_proba = np.array([[1, np.nan], [0.1, 0]])
    for objective in all_automl_objectives.values():
        if objective.score_needs_proba:
            with pytest.raises(ValueError, match="y_predicted contains NaN or infinity"):
                objective.score(y_true, y_predicted_proba)


def test_input_contains_inf():
    y_predicted = np.array([np.inf, 0, 0])
    y_true = np.array([1, 0, 0])
    for objective in all_automl_objectives.values():
        with pytest.raises(ValueError, match="y_predicted contains NaN or infinity"):
            objective.score(y_true, y_predicted)

    y_true = np.array([np.inf, 0, 0])
    y_predicted = np.array([1, 0, 0])
    for objective in all_automl_objectives.values():
        with pytest.raises(ValueError, match="y_true contains NaN or infinity"):
            objective.score(y_true, y_predicted)

    y_true = np.array([1, 0])
    y_predicted_proba = np.array([[1, np.inf], [0.1, 0]])
    for objective in all_automl_objectives.values():
        if objective.score_needs_proba:
            with pytest.raises(ValueError, match="y_predicted contains NaN or infinity"):
                objective.score(y_true, y_predicted_proba)


def test_different_input_lengths():
    y_predicted = np.array([0, 0])
    y_true = np.array([1])
    for objective in all_automl_objectives.values():
        with pytest.raises(ValueError, match="Inputs have mismatched dimensions"):
            objective.score(y_true, y_predicted)

    y_true = np.array([0, 0])
    y_predicted = np.array([1, 2, 0])
    for objective in all_automl_objectives.values():
        with pytest.raises(ValueError, match="Inputs have mismatched dimensions"):
            objective.score(y_true, y_predicted)


def test_zero_input_lengths():
    y_predicted = np.array([])
    y_true = np.array([])
    for objective in all_automl_objectives.values():
        with pytest.raises(ValueError, match="Length of inputs is 0"):
            objective.score(y_true, y_predicted)


def test_probabilities_not_in_0_1_range():
    y_predicted = np.array([0.3, 1.001, 0.3])
    y_true = np.array([1, 0, 1])
    for objective in all_automl_objectives.values():
        if objective.score_needs_proba:
            with pytest.raises(ValueError, match="y_predicted contains probability estimates"):
                objective.score(y_true, y_predicted)

    y_predicted = np.array([0.3, -0.001, 0.3])
    y_true = np.array([1, 0, 1])
    for objective in all_automl_objectives.values():
        if objective.score_needs_proba:
            with pytest.raises(ValueError, match="y_predicted contains probability estimates"):
                objective.score(y_true, y_predicted)

    y_true = np.array([1, 0])
    y_predicted_proba = np.array([[1, 3], [0.1, 0]])
    for objective in all_automl_objectives.values():
        if objective.score_needs_proba:
            with pytest.raises(ValueError, match="y_predicted contains probability estimates"):
                objective.score(y_true, y_predicted_proba)


def test_negative_with_log():
    y_predicted = np.array([-1, 10, 30])
    y_true = np.array([-1, 0, 1])
    for objective in [MeanSquaredLogError(), RootMeanSquaredLogError()]:
        with pytest.raises(ValueError, match="Mean Squared Logarithmic Error cannot be used when targets contain negative values."):
            objective.score(y_true, y_predicted)


def test_binary_more_than_two_unique_values():
    y_predicted = np.array([0, 1, 2])
    y_true = np.array([1, 0, 1])
    for objective in all_automl_objectives.values():
        if isinstance(objective, BinaryClassificationObjective) and not objective.score_needs_proba:
            with pytest.raises(ValueError, match="y_predicted contains more than two unique values"):
                objective.score(y_true, y_predicted)

    y_true = np.array([0, 1, 2])
    y_predicted = np.array([1, 0, 1])
    for objective in all_automl_objectives.values():
        if isinstance(objective, BinaryClassificationObjective) and not objective.score_needs_proba:
            with pytest.raises(ValueError, match="y_true contains more than two unique values"):
                objective.score(y_true, y_predicted)


def test_accuracy_binary():
    obj = AccuracyBinary()
    assert obj.score(np.array([0, 0, 1, 1]),
                     np.array([1, 1, 0, 0])) == pytest.approx(0.0, EPS)
    assert obj.score(np.array([0, 0, 1, 1]),
                     np.array([0, 1, 0, 1])) == pytest.approx(0.5, EPS)
    assert obj.score(np.array([0, 0, 1, 1]),
                     np.array([0, 0, 1, 1])) == pytest.approx(1.0, EPS)


def test_accuracy_multi():
    obj = AccuracyMulticlass()
    assert obj.score(np.array([0, 0, 1, 1]),
                     np.array([1, 1, 0, 0])) == pytest.approx(0.0, EPS)
    assert obj.score(np.array([0, 0, 1, 1]),
                     np.array([0, 1, 0, 1])) == pytest.approx(0.5, EPS)
    assert obj.score(np.array([0, 0, 1, 1]),
                     np.array([0, 0, 1, 1])) == pytest.approx(1.0, EPS)
    assert obj.score(np.array([0, 0, 1, 1, 2, 2]),
                     np.array([0, 0, 0, 0, 0, 0])) == pytest.approx(1 / 3.0, EPS)
    assert obj.score(np.array([0, 0, 0, 0, 0, 0]),
                     np.array([0, 0, 1, 1, 2, 2])) == pytest.approx(1 / 3.0, EPS)


def test_balanced_accuracy_binary():
    obj = BalancedAccuracyBinary()
    assert obj.score(np.array([0, 1, 0, 0, 1, 0]),
                     np.array([0, 1, 0, 0, 0, 1])) == pytest.approx(0.625, EPS)

    assert obj.score(np.array([0, 1, 0, 0, 1, 0]),
                     np.array([0, 1, 0, 0, 1, 0])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([0, 1, 0, 0, 1, 0]),
                     np.array([1, 0, 1, 1, 0, 1])) == pytest.approx(0.0, EPS)


def test_balanced_accuracy_multi():
    obj = BalancedAccuracyMulticlass()
    assert obj.score(np.array([0, 1, 2, 0, 1, 2, 3]),
                     np.array([0, 0, 2, 0, 0, 2, 3])) == pytest.approx(0.75, EPS)

    assert obj.score(np.array([0, 1, 2, 0, 1, 2, 3]),
                     np.array([0, 1, 2, 0, 1, 2, 3])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([0, 1, 2, 0, 1, 2, 3]),
                     np.array([1, 0, 3, 1, 2, 1, 0])) == pytest.approx(0.0, EPS)


def test_f1_binary():
    obj = F1()
    assert obj.score(np.array([0, 1, 0, 0, 1, 0]),
                     np.array([0, 1, 0, 0, 0, 1])) == pytest.approx(0.5, EPS)

    assert obj.score(np.array([0, 1, 0, 0, 1, 1]),
                     np.array([0, 1, 0, 0, 1, 1])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([0, 0, 0, 0, 1, 0]),
                     np.array([0, 1, 0, 0, 0, 1])) == pytest.approx(0.0, EPS)

    assert obj.score(np.array([0, 0]),
                     np.array([0, 0])) == pytest.approx(0.0, EPS)


def test_f1_micro_multi():
    obj = F1Micro()
    assert obj.score(np.array([0, 0, 0, 0, 0, 0, 0, 0, 0]),
                     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])) == pytest.approx(1 / 3.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([2, 2, 2, 0, 0, 0, 1, 1, 1]),
                     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])) == pytest.approx(0.0, EPS)

    assert obj.score(np.array([1, 2]),
                     np.array([0, 0])) == pytest.approx(0.0, EPS)


def test_f1_macro_multi():
    obj = F1Macro()
    assert obj.score(np.array([0, 0, 0, 0, 0, 0, 0, 0, 0]),
                     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])) \
        == pytest.approx(2 * (1 / 3.0) * (1 / 9.0) / (1 / 3.0 + 1 / 9.0), EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([2, 2, 2, 0, 0, 0, 1, 1, 1]),
                     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])) == pytest.approx(0.0, EPS)

    assert obj.score(np.array([1, 2]),
                     np.array([0, 0])) == pytest.approx(0.0, EPS)


def test_f1_weighted_multi():
    obj = F1Weighted()
    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 0, 0, 0, 0, 0, 0])) \
        == pytest.approx(2 * (1 / 3.0) * (1 / 9.0) / (1 / 3.0 + 1 / 9.0), EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([2, 2, 2, 0, 0, 0, 1, 1, 1])) == pytest.approx(0.0, EPS)

    assert obj.score(np.array([0, 0]),
                     np.array([1, 2])) == pytest.approx(0.0, EPS)


def test_precision_binary():
    obj = Precision()
    assert obj.score(np.array([1, 1, 1, 1, 1, 1]),
                     np.array([0, 0, 0, 1, 1, 1])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1]),
                     np.array([1, 1, 1, 1, 1, 1])) == pytest.approx(0.5, EPS)

    assert obj.score(np.array([0, 0, 0, 0, 0, 0]),
                     np.array([1, 1, 1, 1, 1, 1])) == pytest.approx(0.0, EPS)

    assert obj.score(np.array([0, 0, 0, 0, 0, 0]),
                     np.array([0, 0, 0, 0, 0, 0])) == pytest.approx(0.0, EPS)


def test_precision_micro_multi():
    obj = PrecisionMicro()
    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 0, 0, 0, 0, 0, 0])) == pytest.approx(1 / 3.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([2, 2, 2, 0, 0, 0, 1, 1, 1])) == pytest.approx(0.0, EPS)

    assert obj.score(np.array([0, 0]),
                     np.array([1, 2])) == pytest.approx(0.0, EPS)


def test_precision_macro_multi():
    obj = PrecisionMacro()
    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 0, 0, 0, 0, 0, 0])) == pytest.approx(1 / 9.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([2, 2, 2, 0, 0, 0, 1, 1, 1])) == pytest.approx(0.0, EPS)

    assert obj.score(np.array([0, 0]),
                     np.array([1, 2])) == pytest.approx(0.0, EPS)


def test_precision_weighted_multi():
    obj = PrecisionWeighted()
    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 0, 0, 0, 0, 0, 0])) == pytest.approx(1 / 9.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([2, 2, 2, 0, 0, 0, 1, 1, 1])) == pytest.approx(0.0, EPS)

    assert obj.score(np.array([0, 0]),
                     np.array([1, 2])) == pytest.approx(0.0, EPS)


def test_recall_binary():
    obj = Recall()
    assert obj.score(np.array([0, 0, 0, 1, 1, 1]),
                     np.array([1, 1, 1, 1, 1, 1])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1]),
                     np.array([0, 0, 0, 0, 0, 0])) == pytest.approx(0.0, EPS)

    assert obj.score(np.array([1, 1, 1, 1, 1, 1]),
                     np.array([0, 0, 0, 1, 1, 1])) == pytest.approx(0.5, EPS)


def test_recall_micro_multi():
    obj = RecallMicro()
    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 0, 0, 0, 0, 0, 0])) == pytest.approx(1 / 3.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([2, 2, 2, 0, 0, 0, 1, 1, 1])) == pytest.approx(0.0, EPS)

    assert obj.score(np.array([0, 0]),
                     np.array([1, 2])) == pytest.approx(0.0, EPS)


def test_recall_macro_multi():
    obj = RecallMacro()
    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 0, 0, 0, 0, 0, 0])) == pytest.approx(1 / 3.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([2, 2, 2, 0, 0, 0, 1, 1, 1])) == pytest.approx(0.0, EPS)

    assert obj.score(np.array([0, 0]),
                     np.array([1, 2])) == pytest.approx(0.0, EPS)


def test_recall_weighted_multi():
    obj = RecallWeighted()
    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 0, 0, 0, 0, 0, 0])) == pytest.approx(1 / 3.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])) == pytest.approx(1.0, EPS)

    assert obj.score(np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]),
                     np.array([2, 2, 2, 0, 0, 0, 1, 1, 1])) == pytest.approx(0.0, EPS)

    assert obj.score(np.array([0, 0]),
                     np.array([1, 2])) == pytest.approx(0.0, EPS)


def test_log_linear_model():
    obj = MeanSquaredLogError()
    root_obj = RootMeanSquaredLogError()

    s1_predicted = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])
    s1_actual = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0])

    s2_predicted = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])
    s2_actual = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])

    s3_predicted = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])
    s3_actual = np.array([2, 2, 2, 0, 0, 0, 1, 1, 1])

    assert obj.score(s1_predicted, s1_actual) == pytest.approx(0.562467324910)
    assert obj.score(s2_predicted, s2_actual) == pytest.approx(0)
    assert obj.score(s3_predicted, s3_actual) == pytest.approx(0.617267976207983)

    assert root_obj.score(s1_predicted, s1_actual) == pytest.approx(np.sqrt(0.562467324910))
    assert root_obj.score(s2_predicted, s2_actual) == pytest.approx(0)
    assert root_obj.score(s3_predicted, s3_actual) == pytest.approx(np.sqrt(0.617267976207983))


def test_mse_linear_model():
    obj = MSE()
    root_obj = RootMeanSquaredError()

    s1_predicted = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])
    s1_actual = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0])

    s2_predicted = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])
    s2_actual = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])

    s3_predicted = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])
    s3_actual = np.array([2, 2, 2, 0, 0, 0, 1, 1, 1])

    assert obj.score(s1_predicted, s1_actual) == pytest.approx(5. / 3.)
    assert obj.score(s2_predicted, s2_actual) == pytest.approx(0)
    assert obj.score(s3_predicted, s3_actual) == pytest.approx(2.)

    assert root_obj.score(s1_predicted, s1_actual) == pytest.approx(np.sqrt(5. / 3.))
    assert root_obj.score(s2_predicted, s2_actual) == pytest.approx(0)
    assert root_obj.score(s3_predicted, s3_actual) == pytest.approx(np.sqrt(2.))


def test_mcc_catches_warnings():
    y_true = [1, 0, 1, 1]
    y_predicted = [0, 0, 0, 0]
    with pytest.warns(RuntimeWarning) as record:
        sk_matthews_corrcoef(y_true, y_predicted)
        assert "invalid value" in str(record[-1].message)
    with pytest.warns(None) as record:
        MCCBinary().objective_function(y_true, y_predicted)
        MCCMulticlass().objective_function(y_true, y_predicted)
        assert len(record) == 0


def test_mape_time_series_model():
    obj = MAPE()

    s1_actual = np.array([0, 0, 1, 1, 1, 1, 2, 0, 2])
    s1_predicted = np.array([0, 1, 0, 1, 1, 2, 1, 2, 0])

    s2_actual = np.array([-1, -2, 1, 3])
    s2_predicted = np.array([1, 2, -1, -3])

    s3_actual = np.array([1, 2, 4, 2, 1, 2])
    s3_predicted = np.array([0, 2, 2, 1, 3, 2])

    with pytest.raises(ValueError, match="Mean Absolute Percentage Error cannot be used when targets contain the value 0."):
        obj.score(s1_actual, s1_predicted)
    assert obj.score(s2_actual, s2_predicted) == pytest.approx(8 / 4 * 100)
    assert obj.score(s3_actual, s3_predicted) == pytest.approx(4 / 6 * 100)
    assert obj.score(pd.Series(s3_actual, index=range(-12, -6)), s3_predicted) == pytest.approx(4 / 6 * 100)
    assert obj.score(pd.Series(s2_actual, index=range(10, 14)),
                     pd.Series(s2_predicted, index=range(20, 24))) == pytest.approx(8 / 4 * 100)


@pytest.mark.parametrize("objective_class", _all_objectives_dict().values())
def test_calculate_percent_difference(objective_class):
    score = 5
    reference_score = 10

    change = ((-1) ** (not objective_class.greater_is_better) * (score - reference_score)) / reference_score
    answer = 100 * change

    assert objective_class.calculate_percent_difference(score, reference_score) == answer
    assert objective_class.perfect_score is not None


@pytest.mark.parametrize("objective_class,nan_value", product(_all_objectives_dict().values(), [None, np.nan]))
def test_calculate_percent_difference_with_nan(objective_class, nan_value):

    assert pd.isna(objective_class.calculate_percent_difference(nan_value, 2))
    assert pd.isna(objective_class.calculate_percent_difference(-1, nan_value))
    assert pd.isna(objective_class.calculate_percent_difference(nan_value, nan_value))

    assert pd.isna(objective_class.calculate_percent_difference(2, 0))


def test_calculate_percent_difference_negative_and_equal_numbers():

    assert CostBenefitMatrix.calculate_percent_difference(score=5, baseline_score=5) == 0

    assert CostBenefitMatrix.calculate_percent_difference(score=-5, baseline_score=-10) == 50
    assert CostBenefitMatrix.calculate_percent_difference(score=-10, baseline_score=-5) == -100
    assert CostBenefitMatrix.calculate_percent_difference(score=-5, baseline_score=10) == -150
    assert CostBenefitMatrix.calculate_percent_difference(score=10, baseline_score=-5) == 300

    # These values are not possible for LogLossBinary but we need them for 100% coverage
    # We might add an objective where lower is better that can take negative values in the future
    assert LogLossBinary.calculate_percent_difference(score=-5, baseline_score=-10) == -50
    assert LogLossBinary.calculate_percent_difference(score=-10, baseline_score=-5) == 100
    assert LogLossBinary.calculate_percent_difference(score=-5, baseline_score=10) == 150
    assert LogLossBinary.calculate_percent_difference(score=10, baseline_score=-5) == -300


def test_calculate_percent_difference_small():
    expected_value = 100 * -1 * np.abs(1e-9 / (1e-9))
    assert np.isclose(ExpVariance.calculate_percent_difference(score=0, baseline_score=1e-9), expected_value, atol=1e-8)
    assert pd.isna(ExpVariance.calculate_percent_difference(score=0, baseline_score=1e-10))
    assert pd.isna(ExpVariance.calculate_percent_difference(score=1e-9, baseline_score=0))
    assert pd.isna(ExpVariance.calculate_percent_difference(score=0, baseline_score=0))

[(43, 'evalml.objectives.utils._all_objectives_dict', '_all_objectives_dict', False, 'from evalml.objectives.utils import _all_objectives_dict, get_non_core_objectives\n'), (48, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (49, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (54, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (55, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (60, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (61, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (69, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (70, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (75, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (76, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (81, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (82, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (90, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (91, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (96, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (97, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (104, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (105, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (112, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (113, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (119, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (120, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (126, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (127, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (135, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (136, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (143, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (144, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (150, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (151, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (159, 'evalml.objectives.AccuracyBinary', 'AccuracyBinary', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (169, 'evalml.objectives.AccuracyMulticlass', 'AccuracyMulticlass', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (183, 'evalml.objectives.BalancedAccuracyBinary', 'BalancedAccuracyBinary', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (195, 'evalml.objectives.BalancedAccuracyMulticlass', 'BalancedAccuracyMulticlass', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (207, 'evalml.objectives.F1', 'F1', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (222, 'evalml.objectives.F1Micro', 'F1Micro', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (237, 'evalml.objectives.F1Macro', 'F1Macro', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (253, 'evalml.objectives.F1Weighted', 'F1Weighted', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (269, 'evalml.objectives.Precision', 'Precision', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (284, 'evalml.objectives.PrecisionMicro', 'PrecisionMicro', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (299, 'evalml.objectives.PrecisionMacro', 'PrecisionMacro', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (314, 'evalml.objectives.PrecisionWeighted', 'PrecisionWeighted', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (329, 'evalml.objectives.Recall', 'Recall', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (341, 'evalml.objectives.RecallMicro', 'RecallMicro', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (356, 'evalml.objectives.RecallMacro', 'RecallMacro', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (371, 'evalml.objectives.RecallWeighted', 'RecallWeighted', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (386, 'evalml.objectives.MeanSquaredLogError', 'MeanSquaredLogError', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (387, 'evalml.objectives.RootMeanSquaredLogError', 'RootMeanSquaredLogError', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (389, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (390, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (392, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (393, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (395, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (396, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (408, 'evalml.objectives.MSE', 'MSE', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (409, 'evalml.objectives.RootMeanSquaredError', 'RootMeanSquaredError', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (411, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (412, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (414, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (415, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (417, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (418, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (442, 'evalml.objectives.MAPE', 'MAPE', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (444, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (445, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (447, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (448, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (450, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (451, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (137, 'evalml.objectives.MeanSquaredLogError', 'MeanSquaredLogError', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (137, 'evalml.objectives.RootMeanSquaredLogError', 'RootMeanSquaredLogError', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (161, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (163, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (165, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (171, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (173, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (175, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (177, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (179, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (185, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (188, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (191, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (197, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (200, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (203, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (209, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (212, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (215, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (218, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (224, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (227, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (230, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (233, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (240, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (243, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (246, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (249, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (256, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (259, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (262, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (265, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (271, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (274, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (277, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (280, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (286, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (289, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (292, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (295, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (301, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (304, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (307, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (310, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (316, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (319, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (322, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (325, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (331, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (334, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (337, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (343, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (346, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (349, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (352, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (358, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (361, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (364, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (367, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (373, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (376, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (379, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (382, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (398, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (399, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (400, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (403, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (420, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (421, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (422, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (425, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (432, 'pytest.warns', 'pytest.warns', False, 'import pytest\n'), (433, 'sklearn.metrics.matthews_corrcoef', 'sk_matthews_corrcoef', True, 'from sklearn.metrics import matthews_corrcoef as sk_matthews_corrcoef\n'), (435, 'pytest.warns', 'pytest.warns', False, 'import pytest\n'), (453, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (455, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (456, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (457, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (459, 'pytest.approx', 'pytest.approx', False, 'import pytest\n'), (486, 'evalml.objectives.CostBenefitMatrix.calculate_percent_difference', 'CostBenefitMatrix.calculate_percent_difference', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (488, 'evalml.objectives.CostBenefitMatrix.calculate_percent_difference', 'CostBenefitMatrix.calculate_percent_difference', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (489, 'evalml.objectives.CostBenefitMatrix.calculate_percent_difference', 'CostBenefitMatrix.calculate_percent_difference', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (490, 'evalml.objectives.CostBenefitMatrix.calculate_percent_difference', 'CostBenefitMatrix.calculate_percent_difference', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (491, 'evalml.objectives.CostBenefitMatrix.calculate_percent_difference', 'CostBenefitMatrix.calculate_percent_difference', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (495, 'evalml.objectives.LogLossBinary.calculate_percent_difference', 'LogLossBinary.calculate_percent_difference', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (496, 'evalml.objectives.LogLossBinary.calculate_percent_difference', 'LogLossBinary.calculate_percent_difference', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (497, 'evalml.objectives.LogLossBinary.calculate_percent_difference', 'LogLossBinary.calculate_percent_difference', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (498, 'evalml.objectives.LogLossBinary.calculate_percent_difference', 'LogLossBinary.calculate_percent_difference', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (502, 'numpy.abs', 'np.abs', True, 'import numpy as np\n'), (503, 'evalml.objectives.ExpVariance.calculate_percent_difference', 'ExpVariance.calculate_percent_difference', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (504, 'evalml.objectives.ExpVariance.calculate_percent_difference', 'ExpVariance.calculate_percent_difference', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (505, 'evalml.objectives.ExpVariance.calculate_percent_difference', 'ExpVariance.calculate_percent_difference', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (506, 'evalml.objectives.ExpVariance.calculate_percent_difference', 'ExpVariance.calculate_percent_difference', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (44, 'evalml.objectives.utils.get_non_core_objectives', 'get_non_core_objectives', False, 'from evalml.objectives.utils import _all_objectives_dict, get_non_core_objectives\n'), (51, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (57, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (72, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (78, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (93, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (99, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (107, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (138, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (160, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (161, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (162, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (163, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (164, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (165, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (170, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (171, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (172, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (173, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (174, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (175, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (176, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (177, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (178, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (179, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (184, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (185, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (187, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (188, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (190, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (191, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (196, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (197, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (199, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (200, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (202, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (203, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (208, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (209, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (211, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (212, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (214, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (215, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (217, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (218, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (223, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (224, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (226, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (227, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (229, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (230, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (232, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (233, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (238, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (239, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (242, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (243, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (245, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (246, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (248, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (249, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (254, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (255, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (258, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (259, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (261, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (262, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (264, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (265, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (270, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (271, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (273, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (274, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (276, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (277, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (279, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (280, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (285, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (286, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (288, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (289, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (291, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (292, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (294, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (295, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (300, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (301, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (303, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (304, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (306, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (307, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (309, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (310, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (315, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (316, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (318, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (319, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (321, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (322, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (324, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (325, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (330, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (331, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (333, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (334, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (336, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (337, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (342, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (343, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (345, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (346, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (348, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (349, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (351, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (352, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (357, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (358, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (360, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (361, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (363, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (364, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (366, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (367, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (372, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (373, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (375, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (376, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (378, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (379, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (381, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (382, 'numpy.array', 'np.array', True, 'import numpy as np\n'), (402, 'numpy.sqrt', 'np.sqrt', True, 'import numpy as np\n'), (404, 'numpy.sqrt', 'np.sqrt', True, 'import numpy as np\n'), (424, 'numpy.sqrt', 'np.sqrt', True, 'import numpy as np\n'), (426, 'numpy.sqrt', 'np.sqrt', True, 'import numpy as np\n'), (462, 'evalml.objectives.utils._all_objectives_dict', '_all_objectives_dict', False, 'from evalml.objectives.utils import _all_objectives_dict, get_non_core_objectives\n'), (64, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (85, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (116, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (123, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (130, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (147, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (154, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (436, 'evalml.objectives.MCCBinary', 'MCCBinary', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (437, 'evalml.objectives.MCCMulticlass', 'MCCMulticlass', False, 'from evalml.objectives import F1, MAPE, MSE, AccuracyBinary, AccuracyMulticlass, BalancedAccuracyBinary, BalancedAccuracyMulticlass, BinaryClassificationObjective, CostBenefitMatrix, ExpVariance, F1Macro, F1Micro, F1Weighted, LogLossBinary, MCCBinary, MCCMulticlass, MeanSquaredLogError, Precision, PrecisionMacro, PrecisionMicro, PrecisionWeighted, Recall, RecallMacro, RecallMicro, RecallWeighted, RootMeanSquaredError, RootMeanSquaredLogError\n'), (474, 'evalml.objectives.utils._all_objectives_dict', '_all_objectives_dict', False, 'from evalml.objectives.utils import _all_objectives_dict, get_non_core_objectives\n')]