Observability/metrics update (#1962)

* Removing logging from the NOOP implementatino

* Simplifying the percentiles option

* Simple logging implementation

Co-authored-by: chris <chris@Profanity.local>

metrics/logging/reporter.go

@@ -0,0 +1,53 @@
+package logging
+
+import (
+	"time"
+
+	log "github.com/micro/go-micro/v3/logger"
+	"github.com/micro/go-micro/v3/metrics"
+)
+
+// Reporter is an implementation of metrics.Reporter:
+type Reporter struct {
+	logger  log.Logger
+	options metrics.Options
+}
+
+// New returns a configured noop reporter:
+func New(opts ...metrics.Option) *Reporter {
+	options := metrics.NewOptions(opts...)
+	logger := log.NewLogger(log.WithFields(convertTags(options.DefaultTags)))
+	logger.Log(log.InfoLevel, "Metrics/Logging - metrics will be logged (at TRACE level)")
+
+	return &Reporter{
+		logger:  logger,
+		options: metrics.NewOptions(opts...),
+	}
+}
+
+// Count implements the metrics.Reporter interface Count method:
+func (r *Reporter) Count(metricName string, value int64, tags metrics.Tags) error {
+	r.logger.Logf(log.TraceLevel, "Count metric: %s", tags)
+	return nil
+}
+
+// Gauge implements the metrics.Reporter interface Gauge method:
+func (r *Reporter) Gauge(metricName string, value float64, tags metrics.Tags) error {
+	r.logger.Logf(log.TraceLevel, "Gauge metric: %s", tags)
+	return nil
+}
+
+// Timing implements the metrics.Reporter interface Timing method:
+func (r *Reporter) Timing(metricName string, value time.Duration, tags metrics.Tags) error {
+	r.logger.Logf(log.TraceLevel, "Timing metric: %s", tags)
+	return nil
+}
+
+// convertTags turns Tags into prometheus labels:
+func convertTags(tags metrics.Tags) map[string]interface{} {
+	labels := make(map[string]interface{})
+	for key, value := range tags {
+		labels[key] = value
+	}
+	return labels
+}

metrics/logging/reporter_test.go

@@ -0,0 +1,51 @@
+package logging
+
+import (
+	"bytes"
+	"testing"
+	"time"
+
+	log "github.com/micro/go-micro/v3/logger"
+	"github.com/micro/go-micro/v3/metrics"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoggingReporter(t *testing.T) {
+
+	// Make a Reporter:
+	reporter := New(metrics.Path("/prometheus"), metrics.DefaultTags(map[string]string{"service": "prometheus-test"}))
+	assert.NotNil(t, reporter)
+	assert.Equal(t, "prometheus-test", reporter.options.DefaultTags["service"])
+	assert.Equal(t, ":9000", reporter.options.Address)
+	assert.Equal(t, "/prometheus", reporter.options.Path)
+
+	// Make a log buffer and create a new logger to use it:
+	logBuffer := new(bytes.Buffer)
+	reporter.logger = log.NewLogger(log.WithLevel(log.TraceLevel), log.WithOutput(logBuffer), log.WithFields(convertTags(reporter.options.DefaultTags)))
+
+	// Check that our implementation is valid:
+	assert.Implements(t, new(metrics.Reporter), reporter)
+
+	// Test tag conversion:
+	tags := metrics.Tags{
+		"tag1": "false",
+		"tag2": "true",
+	}
+	convertedTags := convertTags(tags)
+	assert.Equal(t, "false", convertedTags["tag1"])
+	assert.Equal(t, "true", convertedTags["tag2"])
+
+	// Test submitting metrics through the interface methods:
+	assert.NoError(t, reporter.Count("test.counter.1", 6, tags))
+	assert.NoError(t, reporter.Count("test.counter.2", 19, tags))
+	assert.NoError(t, reporter.Count("test.counter.1", 5, tags))
+	assert.NoError(t, reporter.Gauge("test.gauge.1", 99, tags))
+	assert.NoError(t, reporter.Gauge("test.gauge.2", 55, tags))
+	assert.NoError(t, reporter.Gauge("test.gauge.1", 98, tags))
+	assert.NoError(t, reporter.Timing("test.timing.1", time.Second, tags))
+	assert.NoError(t, reporter.Timing("test.timing.2", time.Minute, tags))
+
+	// Test reading back the metrics from the logbuffer (doesn't seem to work because the output still goes to StdOut):
+	// assert.Contains(t, logBuffer.String(), "level=debug service=prometheus-test Count metric: map[tag1:false tag2:true]")
+}

metrics/noop/reporter.go

@@ -3,7 +3,6 @@ package noop
 import (
 	"time"
 
-	log "github.com/micro/go-micro/v3/logger"
 	"github.com/micro/go-micro/v3/metrics"
 )
 
@@ -14,8 +13,6 @@ type Reporter struct {
 
 // New returns a configured noop reporter:
 func New(opts ...metrics.Option) *Reporter {
-	log.Info("Metrics/NoOp - not doing anything")
-
 	return &Reporter{
 		options: metrics.NewOptions(opts...),
 	}

metrics/options.go

@@ -5,8 +5,8 @@ var (
 	defaultPrometheusListenAddress = ":9000"
 	// This is the endpoint where the Prometheus metrics will be made available ("/metrics" is the default with Prometheus):
 	defaultPath = "/metrics"
-	// timingObjectives is the default spread of stats we maintain for timings / histograms:
-	defaultTimingObjectives = map[float64]float64{0.0: 0, 0.5: 0.05, 0.75: 0.04, 0.90: 0.03, 0.95: 0.02, 0.98: 0.001, 1: 0}
+	// defaultPercentiles is the default spread of percentiles/quantiles we maintain for timings / histogram metrics:
+	defaultPercentiles = []float64{0, 0.5, 0.75, 0.90, 0.95, 0.98, 0.99, 1}
 )
 
 // Option powers the configuration for metrics implementations:
@@ -14,19 +14,19 @@ type Option func(*Options)
 
 // Options for metrics implementations:
 type Options struct {
-	Address          string
-	Path             string
-	DefaultTags      Tags
-	TimingObjectives map[float64]float64
+	Address     string
+	DefaultTags Tags
+	Path        string
+	Percentiles []float64
 }
 
 // NewOptions prepares a set of options:
 func NewOptions(opt ...Option) Options {
 	opts := Options{
-		Address:          defaultPrometheusListenAddress,
-		DefaultTags:      make(Tags),
-		Path:             defaultPath,
-		TimingObjectives: defaultTimingObjectives,
+		Address:     defaultPrometheusListenAddress,
+		DefaultTags: make(Tags),
+		Path:        defaultPath,
+		Percentiles: defaultPercentiles,
 	}
 
 	for _, o := range opt {
@@ -57,9 +57,9 @@ func DefaultTags(value Tags) Option {
 	}
 }
 
-// TimingObjectives defines the desired spread of statistics for histogram / timing metrics:
-func TimingObjectives(value map[float64]float64) Option {
+// Percentiles defines the desired spread of statistics for histogram / timing metrics:
+func Percentiles(value []float64) Option {
 	return func(o *Options) {
-		o.TimingObjectives = value
+		o.Percentiles = value
 	}
 }

metrics/options_test.go

@@ -9,10 +9,16 @@ import (
 func TestOptions(t *testing.T) {
 
 	// Make some new options:
-	options := NewOptions(Path("/prometheus"), DefaultTags(map[string]string{"service": "prometheus-test"}))
+	options := NewOptions(
+		Address(":9999"),
+		DefaultTags(map[string]string{"service": "prometheus-test"}),
+		Path("/prometheus"),
+		Percentiles([]float64{0.11, 0.22, 0.33}),
+	)
 
 	// Check that the defaults and overrides were accepted:
-	assert.Equal(t, ":9000", options.Address)
-	assert.Equal(t, "/prometheus", options.Path)
+	assert.Equal(t, ":9999", options.Address)
 	assert.Equal(t, "prometheus-test", options.DefaultTags["service"])
+	assert.Equal(t, "/prometheus", options.Path)
+	assert.Equal(t, []float64{0.11, 0.22, 0.33}, options.Percentiles)
 }

metrics/prometheus/metric_family.go

@@ -19,13 +19,22 @@ type metricFamily struct {
 
 // newMetricFamily returns a new metricFamily (useful in case we want to change the structure later):
 func (r *Reporter) newMetricFamily() metricFamily {
+
+	// Take quantile thresholds from our pre-defined list:
+	timingObjectives := make(map[float64]float64)
+	for _, percentile := range r.options.Percentiles {
+		if quantileThreshold, ok := quantileThresholds[percentile]; ok {
+			timingObjectives[percentile] = quantileThreshold
+		}
+	}
+
 	return metricFamily{
 		counters:           make(map[string]*prometheus.CounterVec),
 		gauges:             make(map[string]*prometheus.GaugeVec),
 		timings:            make(map[string]*prometheus.SummaryVec),
 		defaultLabels:      r.convertTags(r.options.DefaultTags),
 		prometheusRegistry: r.prometheusRegistry,
-		timingObjectives:   r.options.TimingObjectives,
+		timingObjectives:   timingObjectives,
 	}
 }
 

metrics/prometheus/reporter.go

@@ -10,6 +10,12 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
+var (
+	// quantileThresholds maps quantiles / percentiles to error thresholds (required by the Prometheus client).
+	// Must be from our pre-defined set [0.0, 0.5, 0.75, 0.90, 0.95, 0.98, 0.99, 1]:
+	quantileThresholds = map[float64]float64{0.0: 0, 0.5: 0.05, 0.75: 0.04, 0.90: 0.03, 0.95: 0.02, 0.98: 0.001, 1: 0}
+)
+
 // Reporter is an implementation of metrics.Reporter:
 type Reporter struct {
 	options            metrics.Options