Increase value precision in histograms from 5e-3 to 1e-12

Previously values could go into lower bucket because of too coarse precision.

Updates https://github.com/VictoriaMetrics/metrics/issues/8
This commit is contained in:
Aliaksandr Valialkin 2020-05-20 02:49:49 +03:00
parent f6ff0f32a2
commit d81dd5ec9c
2 changed files with 7 additions and 1 deletions

View File

@ -14,7 +14,7 @@ const (
decimalMultiplier = 2
bucketSize = 9 * decimalMultiplier
bucketsCount = e10Max - e10Min
decimalPrecision = 0.01 / decimalMultiplier
decimalPrecision = 1e-12
)
// Histogram is a histogram for non-negative values with automatically created buckets.

View File

@ -55,6 +55,12 @@ func TestGetBucketIdxAndOffset(t *testing.T) {
f((1+prec)*math.Pow10(e10Max), -1, 2)
f((1+3*step+prec)*math.Pow10(e10Max), -1, 2)
f(math.Inf(1), -1, 2)
f(999, 11, 17)
f(1000, 11, 17)
f(1001, 12, 0)
f(1002, 12, 0)
f(1003, 12, 0)
}
func TestGetVMRange(t *testing.T) {