export confusion_matrix, class_confusion, visualize, classification_report, condition_positive, condition_negative, predicted_positive,predicted_negative, correctly_classified, incorrectly_classified, sensitivity_score, recall_score, specificity_score, precision_score, positive_predictive_value, accuracy_score, balanced_accuracy_score, negative_predictive_value, false_negative_rate, false_positive_rate, false_discovery_rate, false_omission_rate, f1_score, prevalence_threshold, threat_score, matthews_correlation_coeff, fowlkes_mallows_index, informedness, markedness, cohen_kappa_score, hamming_loss, jaccard_score, confusion_params

using Plots: heatmap
using Statistics: mean

"""
    confusion_params(matrix::Array{Number,2})

Return the true positives, true negatives, false positives, false negatives arrays
from the given n x n matrix. If the provided matrix is not n x n, an assertion
exception: "Given matrix is not n x n" will be raised. As a visualization for the inner
calculation of the function, [this page](https://devopedia.org/images/article/208/6541.1566280388.jpg) may be visited

"""
function confusion_params(matrix::Array{Number,2})
    @assert size(matrix)[1] == size(matrix)[2] "Given matrix is not n x n "
    tp = []; tn = []; fp = []; fn = []
    matrix_sum = sum(matrix)
     @inbounds for i in 1:size(matrix)[1]
        push!(tp, matrix[i,i])
        push!(fn, sum(matrix[i,:]) - tp[i] )
        push!(fp, sum(matrix[:,i]) -tp[i])
        push!(tn, (matrix_sum - tp[i] - fn[i] - fp[i]))
    end
    return tp, tn, fp, fn
end

function check_index(x, none_accepted; class_name = nothing, ith_class = nothing)
    if !none_accepted; @assert class_name != nothing || ith_class != nothing "No class name or class indexing value provided"; end
    if none_accepted && class_name == nothing == ith_class
        return -1
    elseif class_name != nothing
        @assert class_name in x "There is no such class in the labels of the given confusion matrix"
        index = findfirst(x -> x == class_name, x)
        return index
    else
        @assert ith_class >= 0 && ith_class <= length(x) "ith_class value is not in range"
        return ith_class
    end
end

function clear_output(x, zero_division)
    if true in [isnan(i) || isinf(i) for i in x]
        if zero_division == "warn" || zero_division == "0"
            if zero_division == "warn"; @warn "Zero division, replacing NaN or Inf with 0"; end;
            if length(x) > 1
                return replace(x, NaN => 0)
            else
                return 0
            end
        else
            if length(x) > 1
                return replace(x, NaN => 1)
            else
                return 1
            end
        end
    else return x
    end
end

"""
A struct for representing confusion matrix and related computations

## Fields
**`true_positives`** : An array that contains the true positive values of each label. For binary case,
`true_positives` is a single value. For multiclass, ith element in the array corresponds to the
`true_positives` of the ith class in the labels list.

**`true_negatives`** : An array that contains the true negative values of each label. For binary case,
`true_negatives` is a single value. For multiclass, ith element in the array corresponds to the
`true_negatives` of the ith class in the labels list.

**`false_positives`** : An array that contains the false positive values of each label. For binary case,
`false_positives` is a single value. For multiclass, ith element in the array corresponds to the
`false_positives` of the ith class in the labels list.

**false_negatives** : An array that contains the false negative values of each label. For binary case,
`false_negatives` is a single value. For multiclass, ith element in the array corresponds to the
`false_negatives` of the ith class in the labels list.

**`matrix`** : an n x n matrix where n is the length of labels. It represents the actual confusion matrix
from which true positives, true negatives, false positives and false negatives are calculated.

**`Labels`** : an array representing the labels which are used for printing and visualization purposes

**`zero division`** :
    \n\t"warn" => all NaN and Inf values are replaced with zeros and user is warned by @warn macro in the
    \tprocess

    \t"0" => all NaN and Inf values are replaced with zeros but the user is not warned by @warn macro in the
    \tprocess

    \t"1" => all NaN and Inf values are replaced with ones but the user is not warned by @warn macro in the
    \tprocess

"""

struct confusion_matrix
    true_positives::Array{Int}
    true_negatives::Array{Int}
    false_positives::Array{Int}
    false_negatives::Array{Int}
    matrix::Array{Number,2}
    Labels::Array{Union{Int,AbstractString}}
    zero_division::String
end

"""
## Constructors

```confusion_matrix(expected::Array{T,1}, predicted::Array{T,1}; <keyword arguments>) where T <: Union{Int, String}```

Return a confusion matrix object constructed by the expected and predicted arrays. Expected and predicted arrays
must be of size (n,1) or or vector type. Lengths of the expected and predicted arrays must be equal; thus,
there should be a prediction and a ground truth for each classification.

## Keywords

    \n**`labels`** : vector-like of shape (n,1), default = nothing
    \nList of labels to index the matrix. If nothing is given, those that appear at least once
        in expected or predicted are used in sorted order.

    \n**`normalize`** : boolean, default = nothing
    \nDetermines whether or not the confusion matrix (matrix field of the produced confusion matrix) will be normalized.

    \n**`sample_weight`** : Number, default = nothing
    \nSample weights which will be filled in the matrix before confusion params function is called

    \n**`zero_division`** : "warn", "0", "1", default = "warn"
    \n_See:_ confusion matrix fields

## Example
\n
```julia-repl
julia> y_true = [1,1,1,2,3,3,1,2,1,1,2,1];

julia> y_pred = [1,3,2,1,2,3,1,1,2,3,2,1];

julia> x = confusion_matrix(y_true, y_pred)
\n┌ Warning: No labels provided, constructing a label set by union of the unique elements in Expected
and Predicted arrays\n

                  1      2      3
            _____________________
                  3      2      2   │1
                  2      1      0   │2     Predicted
                  0      1      1   │3

julia> y_true = ["emirhan", "knet", "metrics", "confusion", "knet", "confusion", "emirhan", "metrics", "confusion"];

julia> y_pred = ["knet", "knet", "confusion", "confusion", "knet", "emirhan", "emirhan", "knet", "confusion"];

julia> x = confusion_matrix(y_true, y_pred, labels = ["emirhan", "knet", "confusion", "metrics"])

Expected

emirhan           knet      confusion        metrics
____________________________________________________________
1              1              0              0   │emirhan
0              2              0              0   │knet
1              0              2              0   │confusion       Predicted
0              1              1              0   │metrics

```
## References
   [1] [Wikipedia entry for the Confusion matrix]
           (https://en.wikipedia.org/wiki/Confusion_matrix)
           (Note: Wikipedia and other references may use a different
           convention for axes)

_See: confusion params function_ \n
_Source:_ [script](https://github.com/emirhan422/KnetMetrics/blob/main/src/metrics/classification_metrics.jl)

"""
function confusion_matrix(expected::Array{T,1}, predicted::Array{T,1}; labels = nothing, normalize = false, sample_weight = 0, zero_division = "warn") where T <: Union{Int, String}
    @assert length(expected) == length(predicted) "Sizes of the expected and predicted values do not match"
    @assert eltype(expected) <: Union{Int, String} &&  eltype(predicted) <: Union{Int, String} "Expected and Predicted arrays must either be integers or strings"
    @assert eltype(expected) == eltype(predicted) "Element types of Expected and Predicted arrays do not match"
    if labels != nothing; @assert length(labels) != 0 "Labels array must contain at least one value"; end;
    @assert zero_division in ["warn", "0", "1"] "Unknown zero division behaviour specification"
    if labels == nothing
        @warn "No labels provided, constructing a label set by union of the unique elements in Expected and Predicted arrays"
        labels = union(unique(expected),unique(predicted))
        if eltype(labels) == Int
            sort!(labels)
        end
    end
    dictionary = Dict()
    for i in 1:length(labels)
        dictionary[labels[i]] = i
    end
    matrix = zeros(Number, length(labels), length(labels))
    if sample_weight != 0
        fill!(matrix, sample_weight)
    end
    @inbounds for i in 1:length(expected)
       matrix[dictionary[expected[i]],dictionary[predicted[i]]] += 1
    end
    tp, tn, fp, fn = confusion_params(matrix)
    if 0 in tp
        @warn "There are elements of value 0 in the true positives array. This may lead to false values for some functions"
    end
    if 0 in tn
        @warn "There are elements of value 0 in the true negatives array. This may lead to false values for some functions"
    end
    if 0 in fp
       @warn "There are elements of value 0 in the false positives array. This may lead to false values for some functions"
    end
    if 0 in fn
        @warn "There are elements of value 0 in the false negatives array. This may lead to false values for some functions"
    end
    if normalize
       matrix = [round(i, digits = 3) for i in LinearAlgebra.normalize(matrix)]
    end
    return confusion_matrix(tp,tn,fp,fn,matrix,labels, zero_division)
end

"""
```class_confusion(c::confusion_matrix; class_name = nothing, ith_class = nothing)```

\nReturn a binary confusion matrix for the class denoted by `class_name` or `ith_class` arguments.

## Keywords

**`ith_class`** : Int, default = nothing
\n\tReturn the binary confusion matrix of the ith class in the Labels array. This will be ignored if class_name is not `nothing`
**`class_name`** : Int, String, default = nothing
\n\tReturn the binary confusion matrix of the class of given value if exists in the Labels array.

## Example
\n
```julia-repl
julia> y_true = [1,1,1,2,3,3,1,2,1,1,2,1];

julia> y_pred = [1,3,2,1,2,3,1,1,2,3,2,1];

julia> x = confusion_matrix(y_true, y_pred)
\n┌ Warning: No labels provided, constructing a label set by union of the unique elements in Expected
and Predicted arrays

julia> class_confusion(x, ith_class = 2)
2×2 Array{Int64,2}:
 1  3
 2  6

julia> class_confusion(x, class_name = 2)
2×2 Array{Int64,2}:
 1  3
 2  6
```
"""
function class_confusion(c::confusion_matrix; class_name = nothing, ith_class = nothing)
    index = check_index(c.Labels, false ,class_name = class_name, ith_class = ith_class)
    return [c.true_positives[index] c.false_positives[index]; c.false_negatives[index] c.true_negatives[index]]
end

"""
```visualize(c::confusion_matrix)```

Visualize the matrix of the given confusion matrix object by heatmap function of the Plots library.
"""
function visualize(c::confusion_matrix)
    converted_labels = []
    for i in c.Labels
        push!(converted_labels, string(i))
    end
    heatmap(converted_labels, converted_labels, c.matrix, c = :dense)
end

function Base.show(io::IO, ::MIME"text/plain", c::confusion_matrix)
    printer = Int(round(size(c.matrix)[1] / 2)) +1
    label_len = maximum([length(string(i)) for i in c.Labels])[1] + 6
    label_size = length(c.Labels)
    println(io, lpad("Expected\n", printer* label_len ))
    println(io, [lpad(i,label_len) for i in c.Labels]...)
    println(io, repeat("_", length(c.Labels) * label_len))
    for i in 1:size(c.matrix)[1]
        println(io,  [lpad(string(i),label_len) for i in c.matrix[i,:]]..., "   │", c.Labels[i], i == printer ? "\tPredicted" : " ")
    end
end

"""
```classification_report(c::confusion_matrix;<keyword arguments>)```

Return all the values listed below if `return_dict` is true. Else, write the values to the given IO element.

Returned dictionary:
```
    "true-positives" => c.true_positives
    "false-positives" => c.false_positives
    "true-negatives" => c.true_negatives
    "false-negatives" => c.false_negatives
    "condition-positive" => condition_positive(c)
    "condition-negative" => condition_negative(c)
    "predicted-positive" => predicted_positive(c)
    "predicted-negative" => predicted_negative(c)
    "correctly-classified" => correctly_classified(c)
    "incorrectly-classified" => incorrectly_classified(c)
    "sensitivity" => sensitivity_score(c)
    "specificity" => specificity_score(c)
    "precision" => precision_score(c)
    "accuracy-score" => accuracy_score(c)
    "balanced Accuracy" => balanced_accuracy(c)
    "positive-predictive-value" =>  positive_predictive_value(c)
    "negative-predictive-value" => negative_predictive_value(c)
    "false-negative-rate"  => false_negative_rate(c)
    "false-positive-rate"  => false_positive_rate(c)
    "false-discovery-rate" => false_discovery_rate(c)
    "false-omission-rate"  => false_omission_rate(c)
    "f1-score" => f1_score(c)
    "prevalence-threshold" => prevalence_threshold(c)
    "threat-score" => threat_score(c)
    "matthews-correlation-coefficient" => matthews_correlation_coeff(c)
    "fowlkes-mallows-index" => fowlkes_mallows_index(c)
    "informedness" => informedness(c)
    "markedness" => markedness(c)
    "jaccard-score-nonaverage" => jaccard_score(c, average = nothing)
    "jaccard-score-microaverage" => jaccard_score(c, average = "micro")
    "hamming-loss" => hamming_loss(c)
    "cohen-kappa-score" => cohen_kappa_score(c)
```

For a sample output to the given IO element, see Example section.

## Keywords

    \n**`io`** : ::IO, default = Base.stdout
    \n\tIO element to write to

    \n**`return_dict`** : default = false
    \n\tReturn a dictionary as specified below if true; print the values specified below if false

    \n**`target_names`** : vector-like, default = nothing
    \n\tIf not nothing, replace the labels of the given confusion matrix object whilst printing

    \n**`digits`** : Int, default = 2
    \n\tDetermines how the rounding procedure will be digitized. If `return_dict` is true, this will be ignored and the values
    will be placed into the dictionary with full precision

## Example

```julia-repl

julia> y_true = [1,1,1,2,3,3,1,2,1,1,2,1];

julia> y_pred = [1,3,2,1,2,3,1,1,2,3,2,1];

julia> x = confusion_matrix(y_true, y_pred)
┌ Warning: No labels provided, constructing a label set by union of the unique elements in Expected and Predicted arrays
└ @ Path
julia> classification_report(x)
Summary:
confusion_matrix
True Positives: [3, 1, 1]
False Positives: [2, 3, 2]
True Negatives: [3, 6, 8]
False Negatives: [4, 2, 1]

Labelwise Statistics

                                  1    2    3
           Condition Positive:  7.0  3.0  2.0
           Condition Negative:  5.0  9.0 10.0
           Predicted Positive:  5.0  4.0  3.0
           Predicted Negative:  7.0  8.0  9.0
         Correctly Classified:  6.0  7.0  9.0
       Incorrectly Classified:  6.0  5.0  3.0
                  Sensitivity: 0.43 0.33  0.5
                  Specificity:  0.6 0.67  0.8
                    Precision:  0.6 0.25 0.33
               Accuracy Score:  0.5 0.58 0.75
            Balanced Accuracy: 0.51  0.5 0.65
    Negative Predictive Value: 0.43 0.75 0.89
          False Negative Rate: 0.57 0.67  0.5
          False Positive Rate:  0.4 0.33  0.2
         False Discovery Rate:  0.4 0.33  0.2
          False Omission Rate: 0.57 0.25 0.11
                     F1 Score:  0.5 0.29  0.4
                Jaccard Score: 0.33 0.17 0.25
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Main Path
         Prevalence Threshold:16.73  Inf 1.05
                 Threat Score: 0.33 0.17 0.25
Matthews Correlation Coefficient 0.03  0.0 0.26
        Fowlkes Mallows Index: 0.51 0.29 0.41
                 Informedness: 0.03  0.0  0.3
                   Markedness:-0.74-0.81 -0.7

General Statistics

              Accuracy Score:   0.1527777777777778
           Cohen Kappa Score:   0.07692307692307698
                Hamming Loss:   0.5833333333333334
               Jaccard Score:   0.2631578947368421
```
"""
function classification_report(c::confusion_matrix; io::IO = Base.stdout, return_dict = false, target_names = nothing, digits = 2)
    if return_dict
        result_dict = Dict()
        result_dict["true-positives"] = c.true_positives
        result_dict["false-positives"] = c.false_positives
        result_dict["true-negatives"] = c.true_negatives
        result_dict["false-negatives"] = c.false_negatives
        result_dict["condition-positive"] = condition_positive(c)
        result_dict["condition-negative"] = condition_negative(c)
        result_dict["predicted-positive"] = predicted_positive(c)
        result_dict["predicted-negative"] = predicted_negative(c)
        result_dict["correctly-classified"] = correctly_classified(c)
        result_dict["incorrectly-classified"] = incorrectly_classified(c)
        result_dict["sensitivity"] = sensitivity_score(c)
        result_dict["specificity"] = specificity_score(c)
        result_dict["precision"] = precision_score(c)
        result_dict["accuracy-score"] = accuracy_score(c)
        result_dict["balanced Accuracy"] = balanced_accuracy(c)
        result_dict["positive-predictive-value"] =  positive_predictive_value(c)
        result_dict["negative-predictive-value"] = negative_predictive_value(c)
        result_dict["false-negative-rate"]  = false_negative_rate(c)
        result_dict["false-positive-rate"]  = false_positive_rate(c)
        result_dict["false-discovery-rate"] = false_discovery_rate(c)
        result_dict["false-omission-rate"]  = false_omission_rate(c )
        result_dict["f1-score"] = f1_score(c)
        result_dict["prevalence-threshold"] = prevalence_threshold(c)
        result_dict["threat-score"] = threat_score(c)
        result_dict["matthews-correlation-coefficient"] = matthews_correlation_coeff(c)
        result_dict["fowlkes-mallows-index"] = fowlkes_mallows_index(c)
        result_dict["informedness"] = informedness(c)
        result_dict["markedness"] = markedness(c)
        result_dict["jaccard-score-nonaverage"] = jaccard_score(c, average = nothing)
        result_dict["jaccard-score-microaverage"] = jaccard_score(c, average = "micro")
        result_dict["hamming-loss"] = jaccard_score(c)
        result_dict["cohen-kappa-score"] = cohen_kappa_score(c)
        return result_dict
    else
        labels = target_names != nothing && length(target_names) == length(c.Labels) ? target_names : c.Labels
        len = maximum([length(string(i)) for i in labels])
        label_size = length(c.Labels)
        label_len = len + digits + 2
        println(io,"Summary:\n", summary(c))
        println(io,"True Positives: ", c.true_positives)
        println(io,"False Positives: ", c.false_positives)
        println(io,"True Negatives: ", c.true_negatives)
        println(io,"False Negatives: ", c.false_negatives)
        println(io,"\n",lpad("Labelwise Statistics", label_len * Int(round(size(c.matrix)[1] / 2)+1)), "\n")
        println(io,lpad(" ", 30), [lpad(i, label_len) for i in labels]...)
        println(io,lpad("Condition Positive:", 30), [lpad(round(i, digits = digits), label_len) for i in condition_positive(c)]...)
        println(io,lpad("Condition Negative:", 30), [lpad(round(i, digits = digits), label_len) for i in condition_negative(c)]...)
        println(io,lpad("Predicted Positive:", 30), [lpad(round(i, digits = digits), label_len) for i in predicted_positive(c)]...)
        println(io,lpad("Predicted Negative:", 30), [lpad(round(i, digits = digits), label_len) for i in predicted_negative(c)]...)
        println(io,lpad("Correctly Classified:", 30), [lpad(round(i, digits = digits), label_len) for i in correctly_classified(c)]...)
        println(io,lpad("Incorrectly Classified:", 30), [lpad(round(i, digits = digits), label_len) for i in incorrectly_classified(c)]...)
        println(io,lpad("Sensitivity:", 30), [lpad(round(i, digits = digits), label_len) for i in sensitivity_score(c)]...)
        println(io,lpad("Specificity:", 30), [lpad(round(i, digits = digits), label_len) for i in specificity_score(c)]...)
        println(io,lpad("Precision:", 30) , [lpad(round(i, digits = digits), label_len) for i in precision_score(c)]...)
        println(io,lpad("Accuracy Score:", 30 ) ,  [lpad(round(accuracy_score(c, ith_class = i), digits = digits), label_len) for i in 1:label_size]...)
        println(io,lpad("Balanced Accuracy:", 30), [lpad(round(i, digits = digits), label_len) for i in balanced_accuracy(c)]...)
        println(io,lpad("Negative Predictive Value:", 30), [lpad(round(i, digits = digits), label_len) for i in negative_predictive_value(c)]...)
        println(io,lpad("False Negative Rate:", 30), [lpad(round(i, digits = digits), label_len) for i in false_negative_rate(c)]...)
        println(io,lpad("False Positive Rate:", 30), [lpad(round(i, digits = digits), label_len) for i in false_positive_rate(c)]...)
        println(io,lpad("False Discovery Rate:", 30), [lpad(round(i, digits = digits), label_len) for i in false_discovery_rate(c)]...)
        println(io,lpad("False Omission Rate:", 30), [lpad(round(i, digits = digits), label_len) for i in false_omission_rate(c)]...)
        println(io,lpad("F1 Score:", 30), [lpad(round(i, digits = digits), label_len) for i in f1_score(c)]...)
        println(io,lpad("Jaccard Score:", 30), [lpad(round(i, digits = digits), label_len) for i in jaccard_score(c, average = nothing)]...)
        println(io,lpad("Prevalence Threshold:", 30), [lpad(round(i, digits = digits), label_len) for i in prevalence_threshold(c)]...)
        println(io,lpad("Threat Score:", 30), [lpad(round(i, digits = digits), label_len) for i in threat_score(c)]...)
        println(io,lpad("Matthews Correlation Coefficient", 30), [lpad(round(i, digits = digits), label_len) for i in matthews_correlation_coeff(c)]...)
        println(io,lpad("Fowlkes Mallows Index:", 30), [lpad(round(i, digits = digits), label_len) for i in fowlkes_mallows_index(c)]...)
        println(io,lpad("Informedness:", 30), [lpad(round(i, digits = digits), label_len) for i in informedness(c)]...)
        println(io,lpad("Markedness:", 30), [lpad(round(i, digits = digits), label_len) for i in markedness(c)]...)
        println(io,"\n",lpad("General Statistics", label_len * Int(round(size(c.matrix)[1] / 2)+1)), "\n")
        println(io, lpad("Accuracy Score:\t",30), accuracy_score(c))
        println(io, lpad("Cohen Kappa Score:\t", 30), cohen_kappa_score(c))
        println(io, lpad("Hamming Loss:\t", 30), hamming_loss(c))
        println(io, lpad("Jaccard Score:\t", 30), jaccard_score(c, average = "micro"))
    end
end

"""
```condition_positive(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return condition positive values of either the whole confusion matrix or the classes specified by `class_name` or `ith_class`
arguments.

    Condition Positives: True Positives + False Negatives

## Keyowrds

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return condition positive values for all the elements in the labels arrays

## Example

First example no indexing:\n\n

```julia-repl
julia> y_true = ["sample",  "knet", "metrics", "function",  "knet", "knet","function"];

julia> y_pred = ["sample",  "knet", "sample", "function",  "knet", "knet","knet"];

julia> x = confusion_matrix(y_true, y_pred);

julia> condition_positive(x)
4-element Array{Int64,1}:
 1
 3
 1
 2

```

Second example value of a specific class:\n\n

```julia-repl
julia> y_true = ["sample",  "knet", "metrics", "function",  "knet", "knet","function"];

julia> y_pred = ["sample",  "knet", "sample", "function",  "knet", "knet","knet"];

julia> x = confusion_matrix(y_true, y_pred);

julia> condition_positive(x, class_name = "knet")
3

```
_See also_ : `confusion_matrix`, `condition_negative`, `predicted_positive`, `predicted_negative`

"""
function condition_positive(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = c.true_positives + c.false_negatives
        return clear_output(x,c.zero_division)
    else
        x = c.true_positives[index] + c.false_negatives[index]
        return clear_output(x,c.zero_division)
    end
end

"""
```condition_negative(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return condition negative values of either the whole confusion matrix or the classes specified by `class_name` or `ith_class`
arguments.

    Condition Negatives: True Negatives + False Positives

## Keywords

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return condition negative values for all the elements in the labels arrays

## Example

First example no indexing:\n\n

```julia-repl
julia> y_pred = [1,2,3,1,4,2,1,2,3,4,1,2,3,4,1,2,2,2,3,1];

julia> y_true = [1,2,1,1,4,2,4,2,2,4,1,2,3,2,1,2,2,2,1,1];

julia> x = confusion_matrix(y_pred, y_true, labels = [1,2,3,4]);
┌ Warning: There are elements of value 0 in the false positives array. This may lead to false values for some functions
└ @ Path
┌ Warning: There are elements of value 0 in the false negatives array. This may lead to false values for some functions
└ @ Path

julia> condition_negative(x)
4-element Array{Int64,1}:
 14
 13
 16
 17

```

Second example value of a specific class:\n\n

```julia-repl
julia> y_pred = [1,2,3,1,4,2,1,2,3,4,1,2,3,4,1,2,2,2,3,1];

julia> y_true = [1,2,1,1,4,2,4,2,2,4,1,2,3,2,1,2,2,2,1,1];

julia> x = confusion_matrix(y_pred, y_true, labels = [1,2,3,4]);
┌ Warning: There are elements of value 0 in the false positives array. This may lead to false values for some functions
└ @ Path
┌ Warning: There are elements of value 0 in the false negatives array. This may lead to false values for some functions
└ @ Path

julia> condition_negative(x, class_name = 3)
16

```

_See also_ : `confusion_matrix`, `condition_positive`, `predicted_positive`, `predicted_negative`

"""
function condition_negative(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = c.true_negatives + c.false_positives
        return clear_output(x,c.zero_division)
    else
        x = c.true_negatives[index] + c.false_positives[index]
        return clear_output(x,c.zero_division)
    end
end

"""
```predicted_positive(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return predicted positive values of either the whole confusion matrix or the classes specified by `class_name` or `ith_class`
arguments.

    Predicted Positives: True Positives + False Positives

## Keywords

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return predicted positive values for all the elements in the labels arrays

## Example

First example no indexing:\n\n

```julia-repl
julia> y_pred = [2, 3, 4, 2, 5, 3, 2, 3, 4, 5, 2, 3, 4, 5, 2, 3, 3, 3, 4, 2];

julia> y_true = [2, 3, 2, 2, 5, 3, 5, 3, 3, 5, 2, 3, 4, 3, 2, 3, 3, 3, 2, 2];

julia> x = confusion_matrix(y_pred, y_true, labels = [2,3,4,5]);
┌ Warning: There are elements of value 0 in the false positives array. This may lead to false values for some functions
└ @ Path
┌ Warning: There are elements of value 0 in the false negatives array. This may lead to false values for some functions
└ @ Path

julia>  predicted_positive(x)
4-element Array{Int64,1}:
 7
 9
 1
 3

```

Second example value of a specific class:\n\n

```julia-repl
julia> y_pred = [2, 3, 4, 2, 5, 3, 2, 3, 4, 5, 2, 3, 4, 5, 2, 3, 3, 3, 4, 2];

julia> y_true = [2, 3, 2, 2, 5, 3, 5, 3, 3, 5, 2, 3, 4, 3, 2, 3, 3, 3, 2, 2];

julia> x = confusion_matrix(y_pred, y_true, labels = [2,3,4,5]);
┌ Warning: There are elements of value 0 in the false positives array. This may lead to false values for some functions
└ @ Path
┌ Warning: There are elements of value 0 in the false negatives array. This may lead to false values for some functions
└ @ Path

julia> predicted_positive(x, class_name = 3)
9
```

_See also_ : `confusion_matrix`, `condition_negative`, `predicted_positive`, `predicted_negative`

"""
function predicted_positive(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = c.true_positives + c.false_positives
        return clear_output(x,c.zero_division)
    else
        x = c.true_positives[index] + c.false_positives[index]
        return clear_output(x,c.zero_division)
    end
end

"""
```predicted_negative(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return predicted negative values of either the whole confusion matrix or the classes specified by `class_name` or `ith_class`
arguments.

    Predicted Negatives: Negatives + False Negatives

## Keywords

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return predicted negative values for all the elements in the labels arrays

## Example

First example no indexing:\n\n

```julia-repl
julia> y_pred = [2, 3, 4, 2, 5, 3, 2, 3, 4, 5, 2, 3, 4, 5, 2, 3, 3, 3, 4, 2];

julia> y_true = [2, 3, 2, 2, 5, 3, 5, 3, 3, 5, 2, 3, 4, 3, 2, 3, 3, 3, 2, 2];

julia> x = confusion_matrix(y_pred, y_true, labels = [2,3,4,5]);
┌ Warning: There are elements of value 0 in the false positives array. This may lead to false values for some functions
└ @ Path
┌ Warning: There are elements of value 0 in the false negatives array. This may lead to false values for some functions
└ @ Path

julia> predicted_negative(x)
4-element Array{Int64,1}:
 13
 11
 19
 17

```

Second example value of a specific class:\n\n

```julia-repl
julia> y_pred = [2, 3, 4, 2, 5, 3, 2, 3, 4, 5, 2, 3, 4, 5, 2, 3, 3, 3, 4, 2];

julia> y_true = [2, 3, 2, 2, 5, 3, 5, 3, 3, 5, 2, 3, 4, 3, 2, 3, 3, 3, 2, 2];

julia> x = confusion_matrix(y_pred, y_true, labels = [2,3,4,5]);
┌ Warning: There are elements of value 0 in the false positives array. This may lead to false values for some functions
└ @ Path
┌ Warning: There are elements of value 0 in the false negatives array. This may lead to false values for some functions
└ @ Path

julia> predicted_negative(x, class_name = 4)
19

```
_See also_ : `confusion_matrix`, `condition_negative`, `predicted_positive`, `condition_positive`

"""
function predicted_negative(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = c.true_negatives + c.false_negatives
        return clear_output(x,c.zero_division)
    else
        x = c.true_negatives[index] + c.false_negatives[index]
        return clear_output(x,c.zero_division)
    end
end

"""
```correctly_classified(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return number of correctly classified instances of either the whole confusion matrix or the classes specified by `class_name` or `ith_class`
arguments.

    Correctly Classified Values: True Positives + True Negatives

## Keywords

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return number of correctly classified instances for all the elements in the labels arrays

## Example

First example no indexing:\n\n

```julia-repl
julia> y_pred = [2, 3, 4, 2, 5, 3, 2, 3, 4, 5, 2, 3, 4, 5, 2, 3, 3, 3, 4, 2];

julia> y_true = [2, 3, 2, 2, 5, 3, 5, 3, 3, 5, 2, 3, 4, 3, 2, 3, 3, 3, 2, 2];

julia> x = confusion_matrix(y_pred, y_true, labels = [2,3,4,5]);
┌ Warning: There are elements of value 0 in the false positives array. This may lead to false values for some functions
└ @ Path
┌ Warning: There are elements of value 0 in the false negatives array. This may lead to false values for some functions
└ @ Path

julia> correctly_classified(x)
4-element Array{Int64,1}:
 17
 18
 17
 18

```

Second example value of a specific class:\n\n

```julia-repl
julia> y_pred = [2, 3, 4, 2, 5, 3, 2, 3, 4, 5, 2, 3, 4, 5, 2, 3, 3, 3, 4, 2];

julia> y_true = [2, 3, 2, 2, 5, 3, 5, 3, 3, 5, 2, 3, 4, 3, 2, 3, 3, 3, 2, 2];

julia> x = confusion_matrix(y_pred, y_true, labels = [2,3,4,5]);
┌ Warning: There are elements of value 0 in the false positives array. This may lead to false values for some functions
└ @ Path
┌ Warning: There are elements of value 0 in the false negatives array. This may lead to false values for some functions
└ @ Path

julia> correctly_classified(x, ith_class = 1)
17
```

_See also_ : `confusion_matrix`, `predicted_negative`, `predicted_positive`, `incorrectly_classified`

"""
function correctly_classified(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = c.true_positives + c.true_negatives
        return clear_output(x,c.zero_division)
    else
        x = c.true_positives[index] + c.true_negatives[index]
        return clear_output(x,c.zero_division)
    end
end

"""
```incorrectly_classified(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return number of incorrectly classified instances of either the whole confusion matrix or the classes specified by `class_name` or `ith_class`
arguments.

    Inorrectly Classified: False Negatives + False Positives

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return number of incorrectly classified instances for all the elements in the labels arrays

## Example

First example no indexing:\n\n

```julia-repl
julia> y_pred = [2, 3, 4, 2, 5, 3, 2, 3, 4, 5, 2, 3, 4, 5, 2, 3, 3, 3, 4, 2];

julia> y_true = [2, 3, 2, 2, 5, 3, 5, 3, 3, 5, 2, 3, 4, 3, 2, 3, 3, 3, 2, 2];

julia> x = confusion_matrix(y_pred, y_true, labels = [2,3,4,5]);
┌ Warning: There are elements of value 0 in the false positives array. This may lead to false values for some functions
└ @ Path
┌ Warning: There are elements of value 0 in the false negatives array. This may lead to false values for some functions
└ @ Path

julia> incorrectly_classified(x)
4-element Array{Int64,1}:
 3
 2
 3
 2

```

Second example value of a specific class:\n\n

```julia-repl
julia> y_pred = [2, 3, 4, 2, 5, 3, 2, 3, 4, 5, 2, 3, 4, 5, 2, 3, 3, 3, 4, 2];

julia> y_true = [2, 3, 2, 2, 5, 3, 5, 3, 3, 5, 2, 3, 4, 3, 2, 3, 3, 3, 2, 2];

julia> x = confusion_matrix(y_pred, y_true, labels = [2,3,4,5]);
┌ Warning: There are elements of value 0 in the false positives array. This may lead to false values for some functions
└ @ Path
┌ Warning: There are elements of value 0 in the false negatives array. This may lead to false values for some functions
└ @ Path

julia> incorrectly_classified(x, ith_class = 2)
2

```

_See also_ : `confusion_matrix`, `predicted_negative`, `predicted_positive`, `correctly_classified`

"""
function incorrectly_classified(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = c.false_positives + c.false_negatives
        return clear_output(x,c.zero_division)
    else
        x = c.false_positives[index] + c.false_negatives[index]
        return clear_output(x,c.zero_division)
    end
end

"""
```sensitivity_score(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return sensitivity (recall) score of either the whole confusion matrix or the classes specified by `class_name` or `ith_class`
arguments.

    The sensitivity (recall) is the ratio ``tp / (tp + fn)`` where ``tp`` is the number of
    true positives and ``fn`` the number of false negatives. The recall is
    intuitively the ability of the classifier to find all the positive samples.
    The best value is 1 and the worst value is 0.

## Keywords

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return sensitivity(recall) score for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia>  x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> sensitivity_score(x)
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Path
5-element Array{Float64,1}:
 0.2
 0.5
 1.0
 0.0
 0.0

julia> sensitivity_score(x, class_name = 1)
0.2
```

_See also_ : `confusion_matrix` , `recall_score` ,  `balanced_accuracy_score`, `specificity_score`
"""
function sensitivity_score(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = c.true_positives ./ condition_positive(c)
        return clear_output(x, c.zero_division)
    else
        x = c.true_positives[index] / condition_positive(c, ith_class = index)
        return clear_output(x, c.zero_division)
    end
end

"""
```recall_score(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return recall(sensitivity) score of either the whole confusion matrix or the classes specified by `class_name` or `ith_class`
arguments.

    The recall (sensitivity) is the ratio ``tp / (tp + fn)`` where ``tp`` is the number of
    true positives and ``fn`` the number of false negatives. The recall is
    intuitively the ability of the classifier to find all the positive samples.
    The best value is 1 and the worst value is 0.

## Keywords

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return recall (sensitivity) score for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia>  x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> recall_score(x)
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Path
5-element Array{Float64,1}:
 0.2
 0.5
 1.0
 0.0
 0.0

julia> recall_score(x, class_name = 1)
0.2
```

_See also_ : `confusion_matrix` , `sensitivity_score` ,  `balanced_accuracy_score`, `specificity_score`

"""
function recall_score(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    return sensitivity_score(c, ith_class = ith_class, class_name = class_name)
end

"""
```specificity_score(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return specificity score of either the whole confusion matrix or the classes specified by `class_name` or `ith_class`
arguments.

    The specificity is the ratio ``tn / (tn + fp)`` where ``tn`` is the number of
    true negatives and ``fp`` the number of false positives. The specificity is
    intuitively the ability of the classifier to find all the negative samples.
    The best value is 1 and the worst value is 0.

## Keywords

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return specificity score for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia>  x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> specificity(x)
5-element Array{Float64,1}:
 0.6
 1.0
 0.7777777777777778
 0.7
 1.0

julia> specificity(x,ith_class = 2)
1.0
```

_See also_ : ```confusion_matrix```, ```sensitivity_score```, ```balanced_accuracy_score```,```recall_score```

"""
function specificity_score(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = c.true_negatives ./ condition_negative(c)
        return clear_output(x, c.zero_division)
    else
        x = c.true_negatives[index] / condition_negative(c, ith_class = index)
        return clear_output(x, c.zero_division)
    end
end

"""
```precision_score(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return precision score of either the whole confusion matrix or the classes specified by `class_name` or `ith_class`
arguments.

    The precision is the ratio ``tp / (tp + fp)`` where ``tp`` is the number of
    true positives and ``fp`` the number of false positives. The precision is
    intuitively the ability of the classifier not to label as positive a sample
    that is negative.

## Keywords

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return precision score for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> precision_score(x)
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Main Path
5-element Array{Float64,1}:
 0.3333333333333333
 1.0
 0.3333333333333333
 0.0
 0.0

julia>  precision_score(x, class_name = 3)
0.3333333333333333

```

_See also_ : ```confusion_matrix```, ```sensitivity_score```, ```balanced_accuracy_score```,```recall_score```

"""
function precision_score(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = c.true_positives ./ (c.true_positives + c.false_positives)
        return clear_output(x, c.zero_division)
    else
        x = c.true_positives[index] / (c.true_positives[index] + c.false_positives[index])
        return clear_output(x,c.zero_division)
    end
end

"""
```positive_predictive_value(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return  score of either the whole confusion matrix or the classes specified by `class_name` or `ith_class`
arguments.

    The positive predictive value is the ratio ``tp / (tp + fp)`` where ``tp`` is the number of
    true positives and ``fp`` the number of false positives. The positive predictive value is
    intuitively the ability of the classifier not to label as positive a sample
    that is negative.

## Keywords

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return positive predictive value for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> positive_predictive_value(x)
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Main Path
5-element Array{Float64,1}:
 0.3333333333333333
 1.0
 0.3333333333333333
 0.0
 0.0

julia> positive_predictive_value(x, ith_class = 3)
0.3333333333333333

```

_See also_ : ```negative_predictive_value```, ```confusion_matrix```, ```sensitivity_score```, ```balanced_accuracy_score```,```recall_score```

"""
function positive_predictive_value(c::confusion_matrix; ith_class = nothing, class_name = nothing)
   return precision_score(c, class_name = class_name, ith_class = ith_class)
end

"""
```accuracy_score(c::confusion_matrix; ith_class = nothing, class_name = nothing, normalize = true, sample_weight = nothing) ```

Return accuracy classification score.

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

**`normalize`** : bool, default= true
        If ``False``, return the number of correctly classified samples.
        Otherwise, return the fraction of correctly classified samples.

**`sample_weight`** : array-like of shape (n_samples,), default=None
        Sample weights.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return positive predictive value for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> accuracy_score(x)
0.36

julia> accuracy_score(x, normalize = false)
3.5999999999999996

```

_See also_ : ```jaccard_score``` ```confusion_matrix```, ```hamming_loss```, ```balanced_accuracy_score```,```recall_score```

"""
function accuracy_score(c::confusion_matrix; ith_class = nothing, class_name = nothing, normalize = true, sample_weight = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        accuracy_array = [accuracy_score(c, ith_class = i) for i in 1:length(c.true_positives)]
        if normalize
            x = sample_weight == nothing ? (sum(accuracy_array) / sum(c.matrix)) : (sum(accuracy_array .* sample_weight) / sum(c.matrix))
            return clear_output(x,c.zero_division)
        else
            x = sample_weight == nothing ? sum(accuracy_array) : dot(accuracy_array, sample_weight)
            return clear_output(x,c.zero_division)
        end
    else
        if normalize
            x = (c.true_positives[index] + c.true_negatives[index] ) / (condition_positive(c, ith_class = index) + condition_negative(c, ith_class = index))
            return clear_output(x, c.zero_division)
        else
            x = (c.true_positives[index])
            return clear_output(x, c.zero_division)
        end
    end
end

"""
```balanced_accuracy_score(c::confusion_matrix; ith_class = nothing, class_name = nothing) ```

Return balanced accuracy classification score.

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return balanced accuracy score for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> balanced_accuracy_score(x)
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Path
5-element Array{Float64,1}:
 0.4
 0.75
 0.8888888888888888
 0.35
 0.5

julia> balanced_accuracy_score(x, ith_class = 3)
0.8888888888888888

```

_See also_ : ```accuracy_score``` ```confusion_matrix```, ```hamming_loss```, ```balanced_accuracy_score```,```recall_score```

[1] Brodersen, K.H.; Ong, C.S.; Stephan, K.E.; Buhmann, J.M. (2010).
       The balanced accuracy and its posterior distribution.
       Proceedings of the 20th International Conference on Pattern
       Recognition, 3121-24.
[2] John. D. Kelleher, Brian Mac Namee, Aoife D'Arcy, (2015).
       `Fundamentals of Machine Learning for Predictive Data Analytics:
       Algorithms, Worked Examples, and Case Studies
       [link](https://mitpress.mit.edu/books/fundamentals-machine-learning-predictive-data-analytics)

"""
function balanced_accuracy_score(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = [(sensitivity_score(c, ith_class = i) +  specificity_score(c, ith_class = i)) / 2 for i in 1:length(c.true_positives)]
        return clear_output(x,c.zero_division)
    else
        x = (sensitivity_score(c,ith_class = index) +  specificity_score(c, ith_class = index)) / 2
        return clear_output(x,c.zero_division)
    end
end

"""

```negative_predictive_value(c::confusion_matrix; ith_class = nothing, class_name = nothing) ```

Return negative predictive value for the specified class(es).

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return negative predictive value for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> negative_predictive_value(x)
5-element Array{Float64,1}:
 0.42857142857142855
 0.8888888888888888
 1.0
 1.0
 0.8

 julia> negative_predictive_value(x, class_name = 1)
 0.42857142857142855
```

_See Also_ :   ```confusion_matrix```, ```accuracy_score```, ```positive_predictive_value```, ```balanced_accuracy_score```

"""
function negative_predictive_value(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = [c.true_negatives[i] / (c.true_negatives[i] + c.false_negatives[i]) for i in 1:length(c.true_positives)]
        return clear_output(x,c.zero_division)
    else
        x = c.true_negatives[index] / (c.true_negatives[index] + c.false_negatives[index])
        return clear_output(x,c.zero_division)
    end
end

"""

```false_negative_rate(c::confusion_matrix; ith_class = nothing, class_name = nothing) ```

Return false negative rate for the specified class(es).

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return false negative rate for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> false_negative_rate(x)
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Path
5-element Array{Float64,1}:
 0.8
 0.5
 0.0
 0.0
 1.0

 julia> false_negative_rate(x, ith_class = 3)
 0.0
 ```

 _See Also_ :   ```confusion_matrix```, ```false_positive_rate```, ```positive_predictive_value```, ```balanced_accuracy_score```

"""
function false_negative_rate(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = [c.false_negatives[i] / condition_positive(c,ith_class = i) for i in 1:length(c.true_positives)]
        return clear_output(x,c.zero_division)
    else
        x = c.false_negatives[index] / condition_positive(c,ith_class = index)
        return clear_output(x,c.zero_division)
    end
end

"""

```false_positive_rate(c::confusion_matrix; ith_class = nothing, class_name = nothing) ```

Return false positive rate for the specified class(es).

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return false positive rate for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> false_positive_rate(x)
5-element Array{Float64,1}:
 0.4
 0.0
 0.2222222222222222
 0.3
 0.0

 julia> false_positive_rate(x, ith_class = 3)
 0.2222222222222222
 ```

 _See Also_ :   ```confusion_matrix```, ```false_negative_rate```, ```positive_predictive_value```, ```balanced_accuracy_score```
"""
function false_positive_rate(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = [c.false_positives[i] / condition_negative(c,ith_class = i) for i in 1:length(c.true_positives)]
        return clear_output(x,c.zero_division)
    else
        x = c.false_positives[index] / condition_negative(c,ith_class = index)
        return clear_output(x,c.zero_division)
    end
end

"""

```false_discovery_rate(c::confusion_matrix; ith_class = nothing, class_name = nothing) ```

Return false discovery rate for the specified class(es).

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return false discovery rate for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> false_discovery_rate(x)
5-element Array{Float64,1}:
 0.4
 0.0
 0.2222222222222222
 0.3
 0.0

julia> false_discovery_rate(x, ith_class = 3)
0.2222222222222222

```
_See Also_ :   ```confusion_matrix```, ```accuracy_score```, ```positive_predictive_value```, ```false_omission_rate```

"""
function false_discovery_rate(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = [c.false_positives[i] / ( c.false_positives[i]  + c.true_negatives[i]) for i in 1:length(c.true_positives)]
        return clear_output(x,c.zero_division)
    else
        x = c.false_positives[index] / ( c.false_positives[index]  + c.true_negatives[index])
        return clear_output(x,c.zero_division)
    end
end

"""
```false_omission_rate(c::confusion_matrix; ith_class = nothing, class_name = nothing) ```

Return false omission rate for the specified class(es).

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return false omission rate for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> false_omission_rate(x)
5-element Array{Float64,1}:
 0.5714285714285714
 0.11111111111111116
 0.0
 0.0
 0.19999999999999996

julia> false_omission_rate(x, ith_class = 5)
0.19999999999999996
```

_See Also_ :   ```confusion_matrix```, ```accuracy_score```, ```positive_predictive_value```, ```false_discovery_rate```
"""
function false_omission_rate(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = [1 - negative_predictive_value(c,ith_class = i) for i in 1:length(c.true_positives)]
        return clear_output(x,c.zero_division)
    else
        x = 1 - negative_predictive_value(c,ith_class = index)
        return clear_output(x,c.zero_division)
    end
end

"""
```f1_score(c::confusion_matrix; ith_class = nothing, class_name = nothing) ```

Return f1 score for the specified class(es).

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return f1 score for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> f1_score(x)
5-element Array{Float64,1}:
 0.25
 0.6666666666666666
 0.5
 0.0
 0.0

julia> f1_score(x, class_name = 2)
0.6666666666666666
```

_See Also_ :   ```confusion_matrix```, ```accuracy_score```, ```recall_score```, ```false_omission_rate```

"""
function f1_score(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = [(2* c.true_positives[i] ) / (2* c.true_positives[i] + c.false_positives[i] + c.false_negatives[i]) for i in 1:length(c.true_positives)]
        return clear_output(x,c.zero_division)
    else
        x = (2* c.true_positives[index] ) / (2* c.true_positives[index] + c.false_positives[index] + c.false_negatives[index])
        return clear_output(x,c.zero_division)
    end
end

"""

```prevalence_threshold(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return prevalence threshold for the specified class(es).

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return prevalence threshold for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> prevalence_threshold(x)
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Path
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Path
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Path
5-element Array{Float64,1}:
 -2.828427124746191
  0.0
  0.0
 -1.8257418583505536
  0.0

julia> prevalence_threshold(x, ith_class = 1)
-2.828427124746191
```

_See Also_ :   ```confusion_matrix```, ```accuracy_score```, ```recall_score```, ```f1_score```

"""
function prevalence_threshold(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = [(sqrt(abs(sensitivity_score(c,ith_class = i) * (-specificity_score(c,ith_class = i) +1) + specificity_score(c,ith_class = i) -1)) / (sensitivity_score(c,ith_class = i) + specificity_score(c,ith_class = i) -1)) for i in 1:length(c.true_positives)]
        return clear_output(x,c.zero_division)
    else
        x = (sqrt(abs(sensitivity_score(c,ith_class = index) * (-specificity_score(c,ith_class = index) +1) + specificity_score(c,ith_class = index) -1)) / (sensitivity_score(c,ith_class = index) + specificity_score(c,ith_class = index) -1))
        return clear_output(x,c.zero_division)
    end
end

"""

```threat_score(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return threat score for the specified class(es).

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return threat score for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> threat_score(x)
5-element Array{Float64,1}:
 0.14285714285714285
 0.5
 0.3333333333333333
 0.0
 0.0

julia> threat_score(x, ith_class = 3)
0.3333333333333333

```
_See Also_ :   ```confusion_matrix```, ```accuracy_score```, ```recall_score```, ```f1_score```

"""
function threat_score(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = [c.true_positives[i] / (c.true_positives[i] + c.false_negatives[i] + c.false_positives[i]) for i in 1:length(c.true_positives)]
        return clear_output(x,c.zero_division)
    else
        x = c.true_positives[index] / (c.true_positives[index] + c.false_negatives[index] + c.false_positives[index])
        return clear_output(x,c.zero_division)
    end
end

"""

```matthews_correlation_coeff(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return Matthew's Correlation Coefficient for the specified class(es).

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return Matthew's Correlation Coefficient for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> matthews_correlation_coeff(x)
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Path
5-element Array{Float64,1}:
 -0.2182178902359924
  0.6666666666666666
  0.5091750772173156
  0.0
  0.0

julia> matthews_correlation_coeff(x, ith_class = 3)
0.5091750772173156

```

_See Also_ :   ```confusion_matrix```, ```accuracy_score```, ```threat_score```, ```f1_score```

"""
function matthews_correlation_coeff(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = [(c.true_positives[i] * c.true_negatives[i] - c.false_positives[i] * c.false_negatives[i]) / sqrt( abs((c.true_positives[i] + c.false_positives[i]) * (c.true_positives[i] + c.false_negatives[i]) *
            (c.true_negatives[i] + c.false_positives[i]) * (c.true_negatives[i] + c.false_negatives[i])))
         for i in 1:length(c.true_positives)]
        return clear_output(x,c.zero_division)
    else
        x = (c.true_positives[index] * c.true_negatives[index] - c.false_positives[index] * c.false_negatives[index]) / sqrt( (c.true_positives[index] + c.false_positives[index]) * (c.true_positives[index] + c.false_negatives[index]) *
            (c.true_negatives[index] + c.false_positives[index]) * (c.true_negatives[index] + c.false_negatives[index]))
        return clear_output(x,c.zero_division)
    end
end

"""

```fowlkes_mallows_index(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return Fowlkes Mallows Index for the specified class(es).

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return Fowlkes Mallows Index for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> fowlkes_mallows_index(x)
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Main Path
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Main Path
5-element Array{Float64,1}:
 0.2581988897471611
 0.7071067811865476
 0.5773502691896257
 0.0
 0.0

julia> fowlkes_mallows_index(x, ith_class = 2)
0.7071067811865476

```

_See Also_ :   ```confusion_matrix```, ```matthews_correlation_coeff```, ```threat_score```, ```f1_score```
"""
function fowlkes_mallows_index(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = [sqrt(positive_predictive_value(c,ith_class = i) * sensitivity_score(c,ith_class = i)) for i in 1:length(c.true_positives)]
        return clear_output(x,c.zero_division)
    else
        x = sqrt(positive_predictive_value(c,ith_class = index) * sensitivity_score(c,ith_class = index))
        return clear_output(x,c.zero_division)
    end
end

"""

```informedness(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return informedness value for the specified class(es).

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return informedness value for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> informedness(x)
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Path
5-element Array{Float64,1}:
 -0.19999999999999996
  0.5
  0.7777777777777777
 -0.30000000000000004
  0.0

julia> informedness(x,ith_class = 3)
0.7777777777777777

```

_See Also_ :   ```confusion_matrix```, ```matthews_correlation_coeff```, ```markedness```, ```f1_score```
"""
function informedness(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = [ ( specificity_score(c,ith_class = i) + sensitivity_score(c,ith_class = i) -1) for i in 1:length(c.true_positives)]
        return clear_output(x,c.zero_division)
    else
        x = specificity_score(c,ith_class = index) + sensitivity_score(c,ith_class = index) -1
        return clear_output(x,c.zero_division)
    end
end

"""

```markedness(c::confusion_matrix; ith_class = nothing, class_name = nothing)```

Return markedness value for the specified class(es).

## Arguments

**`ith_class`** : Int, default = nothing
Return the results for the ith class in the ith label of the label list of the given confusion matrix object.

**`class_name`** : Int/String, default = nothing
Return the results for the class of the speicifed value in the ith label of the label list of the given confusion matrix object.

If both `class_name` and `ith_class` arguments are equal to `nothing`, return  markedness value for all the elements in the labels arrays

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> markedness(x)
┌ Warning: Zero division, replacing NaN or Inf with 0
└ @ Path
5-element Array{Float64,1}:
 -0.8571428571428572
 -0.11111111111111116
 -0.6666666666666667
 -1.0
 -1.0

julia> markedness(x, ith_class = 1)
-0.19999999999999996

```

_See Also_ :   ```confusion_matrix```, ```matthews_correlation_coeff```, ```informedness```, ```f1_score```

"""
function markedness(c::confusion_matrix; ith_class = nothing, class_name = nothing)
    index = check_index(c.Labels, true, ith_class = ith_class, class_name = class_name)
    if index == -1
        x = [( precision_score(c,ith_class = i) * negative_predictive_value(c,ith_class = i) -1) for i in 1:length(c.true_positives)]
        return clear_output(x,c.zero_division)
    else
        x =  specificity_score(c,ith_class = index) + sensitivity_score(c,ith_class = index) -1
        return clear_output(x,c.zero_division)
    end
end

"""
```cohen_kappa_score(c::confusion_matrix; weights = nothing) ```

Return Cohen's Kappa (a statistic that measures inter-annotator agreement)

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> cohen_kappa_score(x)
0.125
```

_See Also_ :   ```confusion_matrix```, ```accuracy_score```, ```jaccard_score```, ```f1_score```

"""
function cohen_kappa_score(c::confusion_matrix; weights = nothing)
#reference: scikitlearn.metrics.classification.cohen_kappa_score
    @assert weights in [nothing, "quadratic", "linear"] "Unknown kappa weighting type"
    w_mat = nothing
    sum0 = sum(c.matrix, dims = 1)
    sum1 = sum(c.matrix, dims = 2)
    expected = (sum1 * sum0) ./ sum(sum0)
    if weights == nothing
        w_mat = ones(length(c.Labels),length(c.Labels))
        for i in 1:length(c.Labels)
            w_mat[i,i] = 0
        end
    else
        w_mat = zeros(length(c.Labels),length(c.Labels))
        w_mat += [i for i in 1:length(c.Labels)]
        if weights == "linear"
            w_mat = abs(w_mat - transpose(w_mat))
        else
            w_mat = (w_mat - transpose(w_mat)) ^2
        end
    end
    x = sum(w_mat .* c.matrix) / sum(w_mat .* expected)
    return clear_output(1- x,c.zero_division)
end

"""

```hamming_loss(c::confusion_matrix) ```

Compute the average Hamming loss.
    The Hamming loss is the fraction of labels that are incorrectly predicted.

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> hamming_loss(x)
0.7
```

_See Also_ :   ```confusion_matrix```, ```accuracy_score```, ```jaccard_score```, ```f1_score```

"""
function hamming_loss(c::confusion_matrix;)
    x = zeros(sum(c.matrix))
    x[1] = sum(c.false_negatives)
    return clear_output(mean(x), c.zero_division)
end

"""

```jaccard_score(c::confusion_matrix; average = "binary", sample_weight = nothing) ```

Compute Jaccard similarity coefficient score
    The Jaccard index [1], or Jaccard similarity coefficient, defined as
    the size of the intersection divided by the size of the union of two label
    sets, is used to compare set of predicted labels for a sample to the
    corresponding set of labels in ``y_true``.

## Keywords

average : string [None, 'binary' (default), 'micro', 'macro', 'samples, 'weighted']
    If ``None``, the scores for each class are returned. Otherwise, this
    determines the type of averaging performed on the data:
    ``binary``:
        Only report results for the class specified by ``pos_label``.
    ``micro``:
        Calculate metrics globally by counting the total true positives,
        false negatives and false positives.
    ``macro``:
        Calculate metrics for each label, and find their unweighted
        mean.  This does not take label imbalance into account.
    ``weighted``:
        Calculate metrics for each label, and find their average, weighted
        by support (the number of true instances for each label). This
        alters 'macro' to account for label imbalance.
    ``samples``:
        Calculate metrics for each instance, and find their average (only
        meaningful for multilabel classification).
sample_weight : array-like of shape (n_samples,), default=None
    Sample weights.

## Examples

```julia-repl
julia> y_true = [3, 1, 1, 2, 1, 2, 1, 5, 1, 5];

julia> y_pred = [3, 1, 4, 1, 4, 2, 3, 4, 3, 1];

julia> x = confusion_matrix(y_true, y_pred, labels= [1,2,3,4,5]);

julia> hamming_loss(x)
0.7
```

## References

[1] [Wikipedia entry for the Jaccard index](https://en.wikipedia.org/wiki/Jaccard_index)

_See Also_ :   ```confusion_matrix```, ```accuracy_score```, ```hamming_loss```, ```f1_score```

"""
function jaccard_score(c::confusion_matrix; average = "binary", sample_weight = nothing)
    @assert average in [nothing, "binary", "weighted", "samples", "micro", "macro"] "Unknown averaging type"
    if sample_weight != nothing @assert length(sample_weight) == length(c.true_positives) "Dimensions of given sample weight does not match the confusion matrix"; end
    numerator = c.true_positives
    denominator =  c.true_positives + c.false_negatives + c.false_positives
    if average == nothing
        x = numerator ./ denominator
        return clear_output(x, c.zero_division)
    elseif average == "micro"
        numerator = sum(numerator)
        denominator = sum(denominator)
        x = numerator ./ denominator
        return clear_output(x, c.zero_division)
    elseif average == "macro"
        numerator = c.true_positives
        denominator =  c.true_positives + c.false_negatives + c.false_positives
        x = numerator ./ denominator
        return clear_output(mean(x), c.zero_division)
    elseif average == "weighted" || average == "samples"
        weights = nothing
        if average == "weighted"
            weights = c.false_negatives + c.true_positives
        elseif average == "samples"
            weights = sample_weight
        end
        score = numerator ./ denominator
        x = [weights[i] * score[i] for i in 1:length(c.Labels)]
        x / length(c.Labels)
        return clear_output(x, c.zero_division)
    else
        x = [(c.false_negatives[i] + c.true_positives[i])/length(c.Labels) for i in 1:length(c.Labels)]
        return clear_output(x, c.zero_division)
    end
end


"""
    KHist(k::Int)

Estimate the probability density of a univariate distribution at `k` approximately
equally-spaced points.

Ref: [https://www.jmlr.org/papers/volume11/ben-haim10a/ben-haim10a.pdf](https://www.jmlr.org/papers/volume11/ben-haim10a/ben-haim10a.pdf)

A difference from the above reference is that the minimum and maximum values are not allowed to merge into another bin.

# Example

    o = fit!(KHist(25), randn(10^6))

    # Approximate statistics
    using Statistics
    mean(o)
    var(o)
    std(o)
    quantile(o)
    median(o)

    using Plots
    plot(o)
"""
struct KHist{T} <: HistogramStat{T}
    bins::Vector{Pair{T, Int}}  # loc => count
    k::Int
    function KHist(bins::Vector{Pair{T, Int}}, k::Int) where {T}
        k > 2 || error("KHist requires >2 bins")
        new{T}(bins, k)
    end
end
KHist(k::Int, typ::Type{T} = Float64) where {T<:Number} = KHist(Pair{T, Int}[], k)
KHist(k::Int, itr) = fit!(KHist(k, eltype(itr)), itr)

nobs(o::KHist) = length(o.bins) < 1 ? 0 : sum(last, o.bins)

xy(o::KHist) = first.(o.bins), last.(o.bins)
edges(o::KHist) = vcat(first(o.bins[1]), midpoints(first.(o.bins)), first(o.bins[end]))
midpoints(o::KHist) = first.(o.bins)
counts(o::KHist) = last.(o.bins)

function Base.push!(o::KHist{T}, p::Pair{T,Int}) where {T}
    bins = o.bins
    insert!(bins, searchsortedfirst(bins, p), p)
    if length(bins) > o.k
        mindiff = Inf
        i = 0
        for (j, (a,b)) in enumerate(neighbors(bins))
            d = first(b) - first(a)
            if d < mindiff && 1 < j < (length(bins) - 1)  # leave endpoints as extrema
                mindiff = d
                i = j
            end
        end
        a = bins[i]
        b = bins[i + 1]
        n = last(a) + last(b)
        bins[i] = smooth(first(a), first(b), last(b) / n) => n
        deleteat!(bins, i + 1)
    end
    o
end

_fit!(o::KHist{T}, y) where {T} = push!(o, y => 1)

value(o::KHist) = (centers=first.(o.bins), counts=last.(o.bins))

_merge!(a::KHist, b::KHist) = foreach(x -> push!(a, x), b.bins)

#-----------------------------------------------------------------------------# pdf/cdf
ecdf(o::KHist) = ecdf(first.(o.bins); weights=fweights(last.(o.bins)))
@deprecate cdf(o::KHist, x) ecdf(o)(x)

# based on linear interpolation
function pdf(o::KHist, x::Number)
    a, b = extrema(o)
    if x < a || x > b
        return 0.0
    elseif x == a
        return last(o.bins[1]) / area(o)
    else
        i = searchsortedfirst(o.bins, x => 0)
        x1, y1 = o.bins[i-1]
        x2, y2 = o.bins[i]
        return smooth(y1, y2, (x-x1) / (x2 - x1)) / area(o)
    end
end

area(o::KHist) = area(value(o)...)
area(x, y) = 0.5 * sum((x[i] - x[i-1]) * (y[i] + y[i-1]) for i in 2:length(x))

#-----------------------------------------------------------------------------# statistics
Base.extrema(o::KHist) = first(o.bins[1]), first(o.bins[end])
function Statistics.mean(o::KHist)
    x, y = xy(o)
    mean(x, fweights(y))
end
function Statistics.var(o::KHist)
    x, y = xy(o)
    var(x, fweights(y); corrected=true)
end
function Statistics.quantile(o::KHist, q=[0, .25, .5, .75, 1])
    x, y = xy(o)
    quantile(x, fweights(y), q)
end
Statistics.median(o::KHist) = quantile(o, .5)

#-----------------------------------------------------------------------------# plot
@recipe function f(o::KHist; normalize=true)
    x, y = xy(o)
    y2 = normalize ? y ./ area(o) : y
    fillrange --> 0
    seriesalpha --> .5
    x, y2
end


####### Scalogram estimator #######

"""
Variance function of the empirical scalogram of B-Spline wavelet
"""
function bspline_scalogram_variance(H::Real, vm::Integer, sclrng::AbstractVector{<:Integer}, N::Integer)
    V = zeros(Float64, length(sclrng))

    for (j, s) in enumerate(sclrng)
        # B-Spline filter: extra 1/sqrt(s) factor is due to the implementation of DCWT
        filter = intscale_bspline_filter(s, vm)/sqrt(s)
        proc = FractionalWaveletNoise(H, filter)
        V[j] = autocov(proc, 0)^2 + sum((1-i/N) * autocov(proc, i)^2 for i=0:N-1)
    end
    return 4/N * V
end

"""

B-Spline scalogram estimator for Hurst exponent and volatility.

# Args
- X: matrix of wavelet coefficients. Each row corresponds to a scale.
- sclrng: scale of wavelet transform. Each number in `sclrng` corresponds to one row in the matrix X.
- v: vanishing moments of the wavelet
- p: power of the scalogram
"""
function bspline_scalogram_estim(X::AbstractMatrix{<:Real}, sclrng::AbstractVector{<:Integer}, vm::Integer; pow::Real=2., method::Symbol=:optim, cmode::Symbol=:causal, reweight::Bool=true)
    # remove columns containing NaN
    idx = findall(vec(.!any(isnan.(X), dims=1)))
    X = X[:,idx] # view(X,:,idx)

    if length(X) == 0
        return (hurst=NaN, σ=NaN, η=NaN, residual=NaN, vars=nothing, optimizer=nothing)
    else
        @assert length(sclrng) == size(X,1) > 1  "Dimension mismatch."
        @assert any(sclrng .% 2 .== 0) && any(sclrng .> 0)  "All scales must be positive even number."
        @assert pow>0  "Moment must be positive."

        xp = pow * log.(sclrng)
        μX = mean(X, dims=2)
        yp = vec(log.(mean(abs.(X.-μX).^pow, dims=2)))
        # hurst, η, res = NaN, NaN, NaN

        # compute the weighting vector:
        # Run first an estimation of Hurst by linear regression, then use this estimate to compute the weighting vector.
        # Note that the reweighting scheme is exact only for pow=2.
        dg = DataFrames.DataFrame(xvar=xp, yvar=yp)
        opm = GLM.lm(@GLM.formula(yvar~xvar), dg)
        coef = GLM.coef(opm)
        η, hurst = coef[1], coef[2] - 1/2  # intercept and slope
        res = GLM.deviance(opm)  # residual

        ws =  if reweight
            bspline_scalogram_variance(0 < hurst < 1 ? hurst : 0.5, vm, sclrng, size(X,2)) .^ (-1)
        else
            ones(length(sclrng))  # uniform weight
        end
        ws ./= sum(ws)

        # estimation of H and η
        if method == :optim
            yc = yp .- mean(yp)
            xc = xp .- mean(xp)

            # original implentation
            # func = h -> sum(ws .* abs.(yc - h*xc).^1)
            func = h -> sum(ws .* (yc - h*xc).^2)
            # Gradient-free constrained optimization
            ɛ = 1e-2  # search hurst in the interval 0.5+[ɛ, 1-ɛ]
            opm = Optim.optimize(func, 0.5+ɛ, 1.5-ɛ, Optim.Brent())
            hurst = Optim.minimizer(opm)[1] - 1/2
            η = mean(yp - (hurst+1/2)*xp)

            # # implementation of Knut with weight
            # ɛ = 1e-2  # search hurst in the interval 0.5+[ɛ, 1-ɛ]
            # wv1 = 1 ./ (sclrng)
            # func = h -> sum(wv1 .* abs.(yc - h*xc).^2)
            # # func = h -> sum(wv1 .* (yc - h*xc).^2)
            # opm = Optim.optimize(func, 0.5+ε, 1.5-ε, Optim.Brent())
            # hurst1 = Optim.minimizer(opm)[1] - 1/2
            # wv2 = 1 ./ (sclrng .^ 3)
            # func = h -> sum(wv2 .* abs.(yc - h*xc).^2)
            # # func = h -> sum(wv2 .* (yc - h*xc).^2)
            # opm = Optim.optimize(func, 0.5+ε, 1.5-ε, Optim.Brent())
            # hurst2 = Optim.minimizer(opm)[1] - 1/2
            # hurst = max(hurst1, hurst2)
            # # println("Hurst1=$(hurst1), Hurst2=$(hurst2)")
            # η = mean(yp - (hurst+1/2)*xp)

            res = sqrt(opm.minimum / length(xp) / var(yp))
        elseif method == :lm  # using GLM package
            dg = DataFrames.DataFrame(xvar=xp, yvar=yp)
            opm = GLM.glm(@GLM.formula(yvar~xvar), dg, GLM.Normal(), GLM.IdentityLink(), wts=sqrt.(ws))
            # opm = GLM.lm(@GLM.formula(yvar~xvar), dg)
            coef = GLM.coef(opm)
            η, hurst = coef[1], coef[2] - 1/2
            res = sqrt(GLM.deviance(opm) / var(yp)) # / length(xp)

        # elseif method == :irls
        #     coef = IRLS(yp, xp, p; maxiter=10^4, tol=10^-4)
        #     hurst = coef[1][1]-1/2
        #     η = coef[2][1]  # returned value is a scalar in a vector form
        #     opm = nothing
        else
            error("Unknown method $(method).")
        end

        cp = normal_moment_factor(pow)  # constant factor depending on p
        σ = try
            A = Aψρ_bspline(0, 1, hurst, vm, cmode)  # kwargs: mode=:center
            exp((η - log(cp) - log(A)*pow/2)/pow)
        catch
            NaN
        end

        return (hurst=hurst, σ=σ, η=η, residual=res, vars=(xp,yp), optimizer=opm)
    end
end

"""
# Args
- X: sample path of fBm, e.g. the log-price
- v: vanishing moments
"""
function bspline_scalogram_estim(X::AbstractVector{<:Real}, sclrng::AbstractVector{<:Integer}, vm::Integer; kwargs...)

    # B-Spline wavelet transform
    W, M = cwt_bspline(X, sclrng, vm, :causal)
    # Wt = [view(W, findall(M[:,n]),n) for n=1:size(W,2)]
    # truncation of boundary points
    t1 = findall(prod(M, dims=2))[1][1]
    t2 = findall(prod(M, dims=2))[end][1]
    Wt = W[t1:t2, :]

    # # Covariance matrix
    # lag = 0
    # Σ = cov(Wt[1:end-lag, :], Wt[lag+1:end,:], dims=1);

    return bspline_scalogram_estim(Wt', sclrng, v; kwargs...)
end

const scalogram_estim = bspline_scalogram_estim


mutable struct ChernMat{T<:Integer}
    B::Matrix{T}
    m::Int64     # Number of constraints
    n::Int64     # Dimension
    rows::Int    # Current number of rows
end

function ChernMat(A::Matrix{T}) where T<:Integer
    m, n = size(A)
    ChernMat([I A'], m, n, n)
end

function remove_row(A::Matrix{T}, k::Int) where T<:Integer
    m, n = size(A)
    B = Array{T}(undef, m - 1, n)
    i = 1
    for j in 1:m
        if j != k
            B[i,:] = A[j,:]
            i+=1
        end
    end
    return B
end

function add_row(A::Matrix{T}, a::Matrix{T}) where T<:Integer
    # Here a is a row vector
    m, n = size(A)
    B = Array{T}(undef, m+1, n)
    B[1:m, 1:n] = A
    B[m+1, :] = a
    return B
end

function pprint(mat::ChernMat)
    @printf("# Rays: %7d\n", mat.rows)
    for i in 1:mat.rows
        for j in 1:mat.n
            @printf("%6d", mat.B[i,j])
        end
        print(" |")
        for j in mat.n+1:mat.n+mat.m
            @printf("%6d", mat.B[i,j])
        end
        print("\n")
    end
end

function chernikova(mat::ChernMat{T}, verbosity::Int = 0) where T<:Integer
    m, n = mat.m, mat.n
    counter = 1
    while true
        if verbosity > 0
            println("Iteration ", counter)
            println("---------------")
            pprint(mat)
        end
        (col, p) = leading_col(mat)
        if col == 0
            if verbosity > 0; println("Generators have been found!"); end;
            return norm_cols(LHS(mat)')
        elseif p == mat.rows
            if verbosity > 0; println("Zero is the unique solution"); end;
            return Array{T}(undef, mat.n, 0)
        end

        # Inspect matrix to find the maximum size of the next
        # if verbosity > 0; println("Using column $(col) which has $(p) negatives"); end;
        pos, nil, neg = get_pos_neg_indices(mat, col)

        if verbosity > 0
            println("Adding constraint $(col-mat.n) (column $(col)) which is currently violated by $(length(neg)) of $(mat.rows) extremal rays")
            println("New basis will consist of a maximum of $(mat.rows) - $(length(neg)) + $(length(pos)) * $(length(neg)) = $(mat.rows - length(neg) + length(pos)*length(neg)) extremal rays")
        end;

        max_rows = length(pos) + length(nil) + length(pos) * length(neg)
        new_B = Array{T}(undef, max_rows, mat.m + mat.n)

        # Handle case of two rows separately
        if mat.rows == 2 && max_rows == 2
            if verbosity > 0; println("Handling special two row case..."); end;
            new_B[1,:] = mat.B[pos[1],:]
            new_row = combine(mat.B, col, pos[1], neg[1])
            if verbosity > 1; println("Candidate row: ", new_row); end;
            if !collinear(vec(mat.B[pos[1],:]), vec(new_row))
                new_B[2,:] = new_row
                mat.rows = 2
                mat.B = new_B
            else
                mat.rows = 1
                mat.B = new_B[1,:]
            end
            counter += 1
            continue
        end

        # Add rows with non-negative intersections
        # with leading column to next matrix
        for (k,i) in enumerate([pos; nil]); new_B[k,:] = mat.B[i,:]; end;
        new_rows = length(pos) + length(nil)

        # Go through pairs of row with positive and negative
        # coefficients in leading column and combine if necessary
        for i1 in pos, i2 in neg
            if combinable(mat, i1, i2, verbosity - 1)
                new_rows += 1
                new_B[new_rows, :] = combine(mat.B, col, i1, i2)
            end
        end
        mat.B = new_B[1:new_rows,:]
        mat.rows = new_rows
        counter += 1

        if verbosity > 0
            @printf("Added %d / %d positive combinations of extremal rays\n",
                    new_rows - length(pos) - length(nil), length(pos) * length(neg))
            println("------------------------------------------------")
        end
    end
end

@doc """
# Description
Calculates the finite generators of the cone formed
by the intersection of a polyhedral cone and the positive quadrant;
that is the cone formed by the solutions
of the following problem:

   Ax >= 0
    x >= 0

This is solved by Chernikova's algorithm:
"Algorithm for finding a general formula for the non-negative solutions of linear inequalities"
by N.V. Chernikova (1964)

# Arguments
* `A::Matrix{Integer}`: Matrix of cone constraints (each column defines a constraint)
* `verbosity`: level of information printed to standard output

# Returns
* `rays::Matrix{Integer}`: a matrix whose columns are the required cone generators
""" ->
function chernikova(A::Matrix{T}, verbosity::Int = 0) where T<:Integer
    mat = ChernMat(A)
    chernikova(mat, verbosity)
end

function chernikova(A::Matrix{T}, verbosity::Int = 0) where T<:Rational
    C = intmat(A)
    chernikova(C, verbosity)
end

function leading_col(mat::ChernMat{T}) where T<:Integer
    p = mat.rows
    I = 0
    for i in mat.n+1:mat.n+mat.m
        r = sum(mat.B[:,i] .< 0.0)
        if r == mat.rows
            return (i, r)
        elseif (r > 0) && (r < p)
           I, p = i, r
        end
    end
    return (I, p)
end

LHS(mat::ChernMat) = mat.B[:, 1:mat.n]

function get_pos_neg_indices(mat::ChernMat{T}, col::Int64) where T<:Integer
    pos, nil, neg = Int[],Int[],Int[]
    for k in 1:mat.rows
        if mat.B[k,col] > 0.0; push!(pos, k)
        elseif mat.B[k,col] < 0.0; push!(neg, k)
        else push!(nil, k) end
    end
    pos, nil, neg
end

function combinable(mat::ChernMat{T}, i1::Int64, i2::Int64, verbosity::Int = 0) where T<:Integer
    # Find all columns for which the rows have a common zero
    # LHS
    zero_cols = Int[]
    for j in 1:mat.n
        if (mat.B[i1,j] == 0.0) && (mat.B[i2,j] == 0.0)
            push!(zero_cols, j)
        end
    end

    # RHS
    for j in mat.n+1:mat.n + mat.m
        if all(mat.B[:,j] .>= 0.0)
            if (mat.B[i1,j] == 0.0) && (mat.B[i2,j] == 0)
                push!(zero_cols, j)
            end
        end
    end

    if length(zero_cols) == 0
        if verbosity > 0; println("Rays $(i1) and $(i2) saturate no common constraint - they do not combine to form a new extremal ray") end
        return false
    end

    if verbosity > 0;
        println("Rays $(i1) and $(i2) have ", length(zero_cols), " are both saturated by constraints ", zero_cols);
    end;

    for i in 1:mat.rows
        if i == i1 || i == i2; continue; end

        if verbosity > 1; println("\tRay $i - values on saturated constraints: ",  mat.B[i,zero_cols]); end;
        if vec(mat.B[i,zero_cols]) == zeros(length(zero_cols))
            if verbosity > 0; println("\tRay $i saturates all saturated constraints of these rays so do not combine") end;
            return false
        end
    end
    if verbosity > 0;
        println("\tNo other ray saturates the common saturated constraints of these rays so combine to form a new extreme ray")
    end;
    return true
end

# function combine(mat::Matrix{T}, col::Int, i1::Int, i2::Int) where T<:Integer
#     a, b = 1.0, -mat[i1,col]/mat[i2,col]
#     return a*mat[i1,:] + b*mat[i2,:]
# end

function combine(mat::Matrix{T}, col::Int, i1::Int, i2::Int) where T<:Integer
    t = gcd(mat[i1, col], mat[i2, col])
    a1, a2 = div(mat[i1, col],t), div(mat[i2, col],t)
    return a1 * mat[i2,:] - a2 * mat[i1,:]
end

function combine(col::Int, vec1::Vector{T}, vec2::Vector{T}) where T<:Integer
    t = gcd(vec1[col], vec2[col])
    a1, a2 = div(vec1[col],t), div(vec2[col],t)
    return a1 * vec2 - a2 * vec1
end


using OrdinaryDiffEq, DiffEqDevTools, DiffEqBase, Test, Random
using DiffEqProblemLibrary.ODEProblemLibrary: importodeproblems; importodeproblems()
import DiffEqProblemLibrary.ODEProblemLibrary: prob_ode_2Dlinear, prob_ode_linear

Random.seed!(100)
## Convergence Testing
dts = (1/2) .^ (7:-1:4)
testTol = 0.2

prob = prob_ode_linear
sol = solve(prob,OwrenZen3())
@test length(sol) < 20
sol = solve(prob,OwrenZen4())
@test length(sol) < 20
sol = solve(prob,OwrenZen5())
@test length(sol) < 20

sim = test_convergence(dts,prob,OwrenZen3(),dense_errors=true)
@test sim.𝒪est[:final] ≈ 3 atol=testTol
@test sim.𝒪est[:L2] ≈ 3 atol=testTol
sim = test_convergence(dts,prob,OwrenZen4(),dense_errors=true)
@test sim.𝒪est[:final] ≈ 4 atol=testTol
@test sim.𝒪est[:L2] ≈ 4 atol=testTol
sim = test_convergence(dts,prob,OwrenZen5(),dense_errors=true)
@test sim.𝒪est[:final] ≈ 5 atol=testTol
@test sim.𝒪est[:L2] ≈ 5 atol=testTol

prob = prob_ode_2Dlinear
sol = solve(prob,OwrenZen3())
@test length(sol) < 20
sol = solve(prob,OwrenZen4())
@test length(sol) < 20
sol = solve(prob,OwrenZen5())
@test length(sol) < 20

sim = test_convergence(dts,prob,OwrenZen3(),dense_errors=true)
@test sim.𝒪est[:final] ≈ 3 atol=testTol
@test sim.𝒪est[:L2] ≈ 3 atol=testTol
sim = test_convergence(dts,prob,OwrenZen4(),dense_errors=true)
@test sim.𝒪est[:final] ≈ 4 atol=testTol
@test sim.𝒪est[:L2] ≈ 4 atol=testTol
sim = test_convergence(dts,prob,OwrenZen5(),dense_errors=true)
@test sim.𝒪est[:final] ≈ 5 atol=testTol
@test sim.𝒪est[:L2] ≈ 5 atol=testTol