stat: add comment for calculating variance in cases with low weight sum
MODIFY
stat/stat.go
stat/stat.go
@@ -1076,6 +1076,7 @@ func empiricalQuantile(p float64, x, weights []float64, sumWeights float64) floa
// Skew computes the skewness of the sample data.
// If weights is nil then all of the weights are 1. If weights is not nil, then
// len(x) must equal len(weights).
+// When weights sum to 1 or less, a biased variance estimator should be used.
func Skew(x, weights []float64) float64 {
mean, std := MeanStdDev(x, weights)
@@ -1195,7 +1196,8 @@ func StdDev(x, weights []float64) float64 {
return std
}
-// MeanStdDev returns the sample mean and standard deviation
+// MeanStdDev returns the sample mean and unbiased standard deviation
+// When weights sum to 1 or less, a biased variance estimator should be used.
func MeanStdDev(x, weights []float64) (mean, std float64) {
mean, variance := MeanVariance(x, weights)
return mean, math.Sqrt(variance)
@@ -1213,21 +1215,23 @@ func StdScore(x, mean, std float64) float64 {
return (x - mean) / std
}
-// Variance computes the weighted sample variance:
+// Variance computes the unbiased weighted sample variance:
// \sum_i w_i (x_i - mean)^2 / (sum_i w_i - 1)
// If weights is nil then all of the weights are 1. If weights is not nil, then
// len(x) must equal len(weights).
+// When weights sum to 1 or less, a biased variance estimator should be used.
func Variance(x, weights []float64) float64 {
_, variance := MeanVariance(x, weights)
return variance
}
-// MeanVariance computes the sample mean and variance, where the mean and variance are
+// MeanVariance computes the sample mean and unbiased variance, where the mean and variance are
// \sum_i w_i * x_i / (sum_i w_i)
// \sum_i w_i (x_i - mean)^2 / (sum_i w_i - 1)
// respectively.
// If weights is nil then all of the weights are 1. If weights is not nil, then
// len(x) must equal len(weights).
+// When weights sum to 1 or less, a biased variance estimator should be used.
func MeanVariance(x, weights []float64) (mean, variance float64) {
// This uses the corrected two-pass algorithm (1.7), from "Algorithms for computing
// the sample variance: Analysis and recommendations" by Chan, Tony F., Gene H. Golub,

stat: add comment for calculating variance in cases with low weight sum